08:12:35.390 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:12:35.390 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskzFzfv3/test-tmp
08:12:35.390 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:12:35.391 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskzFzfv3/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:12:35.394 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:12:35.709 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:12:35.710 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:12:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:35.713016 12644 mini_oidc.cc:150] Starting JWKS server
08:12:35.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:35.716619 12644 webserver.cc:469] Webserver started at http://127.0.0.1:36179/ using document root <none> and password file <none>
08:12:35.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:35.717047 12644 mini_oidc.cc:186] Starting OIDC Discovery server
08:12:35.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:35.718732 12644 webserver.cc:469] Webserver started at http://127.0.0.1:36731/ using document root <none> and password file <none>
08:12:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T08:12:35Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:12:35.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T08:12:35Z Disabled control of system clock
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:35.781152 12644 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/wal
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/data
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/logs
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/data/info.pb
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:12:35.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.62:45363
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.62
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:35.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36179/jwks.json with env {}
08:12:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.092553 12656 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:12:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.093111 12656 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:12:36.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.093649 12656 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:12:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.125551 12656 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:12:36.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.125864 12656 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:12:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.126070 12656 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:12:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.126273 12656 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:12:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.126462 12656 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:12:36.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.126608 12656 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36179/jwks.json
08:12:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:36.162339 12656 master_runner.cc:386] Master server non-default flags:
08:12:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/data
08:12:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/wal
08:12:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:36.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.62:45363
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36179/jwks.json
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/data/info.pb
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.62
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/logs
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:12:36.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:36.163651 12656 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:12:36.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:36.165275 12656 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:12:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.179142 12662 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:37.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.180248 12665 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:37.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:36.180965 12663 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:37.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:36.180974 12656 server_base.cc:1048] running on GCE node
08:12:37.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.283500 12656 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:12:37.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.286566 12656 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:12:37.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.287994 12656 hybrid_clock.cc:648] HybridClock initialized: now 1750839157287947 us; error 64 us; skew 500 ppm
08:12:37.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.288739 12656 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:12:37.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.295226 12656 webserver.cc:469] Webserver started at http://127.12.89.62:39263/ using document root <none> and password file <none>
08:12:37.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.296130 12656 fs_manager.cc:362] Metadata directory not provided
08:12:37.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.296317 12656 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:12:37.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.296713 12656 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:12:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.301259 12656 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/data/instance:
08:12:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5097db0d65045d5926ad111e5e987c2"
08:12:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:37 on dist-test-slave-nz51"
08:12:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.302409 12656 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/wal/instance:
08:12:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5097db0d65045d5926ad111e5e987c2"
08:12:37.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:37 on dist-test-slave-nz51"
08:12:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.310230 12656 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
08:12:37.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.315659 12672 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:12:37.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.316799 12656 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
08:12:37.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.317166 12656 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/wal
08:12:37.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f5097db0d65045d5926ad111e5e987c2"
08:12:37.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:37 on dist-test-slave-nz51"
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.317569 12656 fs_report.cc:389] FS layout report
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/wal
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/wal
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/data/data
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:12:37.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:12:37.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.377552 12656 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:12:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.378957 12656 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:12:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.379340 12656 kserver.cc:163] Server-wide thread pool size limit: 3276
08:12:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.446731 12656 rpc_server.cc:307] RPC server started. Bound to: 127.12.89.62:45363
08:12:37.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.446799 12723 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.89.62:45363 every 8 connection(s)
08:12:37.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.449326 12656 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/data/info.pb
08:12:37.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.454742 12724 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:12:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.455634 12644 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 12656
08:12:37.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.456158 12644 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-0/wal/instance
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.461195 12644 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/wal
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/data
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/logs
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/data/info.pb
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:37.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.61:35831
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.61
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:37.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36179/jwks.json with env {}
08:12:37.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.469542 12724 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } has no permanent_uuid. Determining permanent_uuid...
08:12:37.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.494928 12724 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } has no permanent_uuid. Determining permanent_uuid...
08:12:37.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.497576 12674 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.89.61:35831: connect: Connection refused (error 111)
08:12:37.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.501608 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.61:35831: Network error: Client connection negotiation failed: client connection to 127.12.89.61:35831: connect: Connection refused (error 111)
08:12:37.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.551074 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } attempt: 1
08:12:37.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.554890 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.61:35831: Network error: Client connection negotiation failed: client connection to 127.12.89.61:35831: connect: Connection refused (error 111)
08:12:37.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.623353 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } attempt: 2
08:12:37.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.627544 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.61:35831: Network error: Client connection negotiation failed: client connection to 127.12.89.61:35831: connect: Connection refused (error 111)
08:12:37.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.718981 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } attempt: 3
08:12:37.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.723469 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.61:35831: Network error: Client connection negotiation failed: client connection to 127.12.89.61:35831: connect: Connection refused (error 111)
08:12:37.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.757149 12726 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:12:37.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.757802 12726 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:12:37.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.758356 12726 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:12:37.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.789137 12726 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:12:37.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.789498 12726 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:12:37.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.789781 12726 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:12:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.790061 12726 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:12:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.790336 12726 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:12:37.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.790534 12726 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36179/jwks.json
08:12:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.825395 12726 master_runner.cc:386] Master server non-default flags:
08:12:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:37.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/data
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/wal
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.61:35831
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36179/jwks.json
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/data/info.pb
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.61
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/logs
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:12:37.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:12:37.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:12:37.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:12:37.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:12:37.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:12:37.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:12:37.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.827072 12726 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:12:37.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.828742 12726 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:12:37.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.842564 12735 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:37.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:37.867012 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } attempt: 4
08:12:37.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.872529 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.61:35831: Network error: Client connection negotiation failed: client connection to 127.12.89.61:35831: connect: Connection refused (error 111)
08:12:38.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:38.172055 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } attempt: 5
08:12:38.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:38.177213 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.61:35831: Network error: Client connection negotiation failed: client connection to 127.12.89.61:35831: connect: Connection refused (error 111)
08:12:38.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:38.724745 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } attempt: 6
08:12:38.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:38.733500 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.61:35831: Network error: Client connection negotiation failed: client connection to 127.12.89.61:35831: connect: Connection refused (error 111)
08:12:38.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:37.843551 12736 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:38.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:38.983196 12738 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:38.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:38.984995 12726 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 1.146s user 0.429s sys 0.713s
08:12:38.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:38.985339 12726 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 1.146s user 0.429s sys 0.713s
08:12:38.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:38.985615 12726 server_base.cc:1048] running on GCE node
08:12:38.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:38.986675 12726 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:12:38.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:38.989209 12726 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:12:38.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:38.990590 12726 hybrid_clock.cc:648] HybridClock initialized: now 1750839158990563 us; error 43 us; skew 500 ppm
08:12:38.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:38.991470 12726 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:12:38.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:38.998871 12726 webserver.cc:469] Webserver started at http://127.12.89.61:34809/ using document root <none> and password file <none>
08:12:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:38.999799 12726 fs_manager.cc:362] Metadata directory not provided
08:12:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.000013 12726 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:12:39.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.000468 12726 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:12:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.005108 12726 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/data/instance:
08:12:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90567ef9b964421fb03b1f82dab9c962"
08:12:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:38 on dist-test-slave-nz51"
08:12:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.006306 12726 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/wal/instance:
08:12:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90567ef9b964421fb03b1f82dab9c962"
08:12:39.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:38 on dist-test-slave-nz51"
08:12:39.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.014003 12726 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
08:12:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.019923 12747 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:12:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.021003 12726 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
08:12:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.021347 12726 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/wal
08:12:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "90567ef9b964421fb03b1f82dab9c962"
08:12:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:38 on dist-test-slave-nz51"
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.021713 12726 fs_report.cc:389] FS layout report
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/wal
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/wal
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/data/data
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:12:39.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:12:39.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.090668 12726 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:12:39.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.092104 12726 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:12:39.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.092506 12726 kserver.cc:163] Server-wide thread pool size limit: 3276
08:12:39.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.160260 12726 rpc_server.cc:307] RPC server started. Bound to: 127.12.89.61:35831
08:12:39.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.160337 12798 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.89.61:35831 every 8 connection(s)
08:12:39.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.162885 12726 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/data/info.pb
08:12:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.166409 12644 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 12726
08:12:39.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.166787 12644 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-1/wal/instance
08:12:39.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.168067 12799 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.168797 12644 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/wal
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/data
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/logs
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/data/info.pb
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.60:39467
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.60
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36179/jwks.json with env {}
08:12:39.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.179504 12799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } has no permanent_uuid. Determining permanent_uuid...
08:12:39.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.199995 12799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } has no permanent_uuid. Determining permanent_uuid...
08:12:39.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.213783 12799 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } has no permanent_uuid. Determining permanent_uuid...
08:12:39.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.216058 12748 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.220237 12799 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.269666 12799 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 1
08:12:39.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.273772 12799 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:39.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.342168 12799 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 2
08:12:39.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.346060 12799 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:39.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.437458 12799 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 3
08:12:39.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.441442 12799 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:39.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.465755 12801 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:12:39.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.466373 12801 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:12:39.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.466938 12801 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:12:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.498392 12801 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:12:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.498728 12801 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:12:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.499027 12801 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:12:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.499295 12801 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:12:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.499557 12801 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:12:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.499773 12801 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36179/jwks.json
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.535168 12801 master_runner.cc:386] Master server non-default flags:
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/data
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/wal
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.60:39467
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:12:39.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36179/jwks.json
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/data/info.pb
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.60
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/logs
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:12:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:12:39.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.536818 12801 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:12:39.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.538513 12801 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:12:39.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.552043 12812 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:39.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.584861 12799 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 4
08:12:39.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.588441 12799 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:39.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.794023 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } attempt: 7
08:12:39.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.804877 12724 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } has no permanent_uuid. Determining permanent_uuid...
08:12:39.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.809831 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:39.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.868356 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 1
08:12:39.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.872339 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:39.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.887840 12799 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 5
08:12:39.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.891896 12799 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:39.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.953816 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 2
08:12:39.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.957811 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:40.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.043267 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 3
08:12:40.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:40.047068 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.187589 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 4
08:12:40.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:40.192569 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:40.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.439296 12799 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 6
08:12:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:40.446230 12799 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:40.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.476049 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 5
08:12:40.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:40.480333 12724 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.12.89.60:39467: Network error: Client connection negotiation failed: client connection to 127.12.89.60:39467: connect: Connection refused (error 111)
08:12:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.552745 12811 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:39.553601 12801 server_base.cc:1048] running on GCE node
08:12:40.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:39.553258 12814 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:40.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.657162 12801 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:12:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.659750 12801 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:12:40.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.661099 12801 hybrid_clock.cc:648] HybridClock initialized: now 1750839160661057 us; error 47 us; skew 500 ppm
08:12:40.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.661926 12801 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:12:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.667972 12801 webserver.cc:469] Webserver started at http://127.12.89.60:37393/ using document root <none> and password file <none>
08:12:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.668893 12801 fs_manager.cc:362] Metadata directory not provided
08:12:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.669247 12801 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:12:40.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.670336 12801 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:12:40.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.675341 12801 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/data/instance:
08:12:40.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065"
08:12:40.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:40 on dist-test-slave-nz51"
08:12:40.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.676415 12801 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/wal/instance:
08:12:40.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065"
08:12:40.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:40 on dist-test-slave-nz51"
08:12:40.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.683362 12801 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.008s
08:12:40.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.688792 12826 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:12:40.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.689827 12801 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
08:12:40.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.690147 12801 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/wal
08:12:40.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065"
08:12:40.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:40 on dist-test-slave-nz51"
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.690472 12801 fs_report.cc:389] FS layout report
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/wal
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/wal
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/data/data
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:12:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:12:40.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.741380 12801 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:12:40.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.742887 12801 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:12:40.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.743342 12801 kserver.cc:163] Server-wide thread pool size limit: 3276
08:12:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.811448 12801 rpc_server.cc:307] RPC server started. Bound to: 127.12.89.60:39467
08:12:40.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.811515 12877 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.89.60:39467 every 8 connection(s)
08:12:40.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.814275 12801 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/data/info.pb
08:12:40.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.819075 12644 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 12801
08:12:40.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.819568 12644 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/master-2/wal/instance
08:12:40.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.820184 12878 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:12:40.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.836880 12878 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } has no permanent_uuid. Determining permanent_uuid...
08:12:40.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.861581 12878 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } has no permanent_uuid. Determining permanent_uuid...
08:12:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.871515 12878 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } has no permanent_uuid. Determining permanent_uuid...
08:12:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.887199 12878 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065: Bootstrap starting.
08:12:40.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.893469 12878 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065: Neither blocks nor log segments found. Creating new log.
08:12:40.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.895153 12878 log.cc:826] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065: Log is configured to *not* fsync() on all Append() calls
08:12:40.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.900498 12878 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065: No bootstrap required, opened a new log
08:12:40.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.918838 12878 raft_consensus.cc:357] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } }
08:12:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.919759 12878 raft_consensus.cc:383] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:12:40.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.920070 12878 raft_consensus.cc:738] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2f4bf3c4ee5d409bb6d19a3abadc5065, State: Initialized, Role: FOLLOWER
08:12:40.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.921008 12878 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } }
08:12:40.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.923434 12887 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } } }
08:12:40.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.924054 12887 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [sys.catalog]: This master's current role is: FOLLOWER
08:12:40.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.924821 12878 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [sys.catalog]: configured and running, proceeding with master startup.
08:12:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:40.952896 12898 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:12:40.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:40.953276 12898 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:12:40.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.943863 12801 master_runner.cc:186] Error getting master registration for 127.12.89.62:45363: OK, instance_id { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" instance_seqno: 1750839157428066 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:12:40.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:40.967175 12801 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:12:40.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:40.998277 12832 tablet.cc:2378] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:12:41.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.033396 12724 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 6
08:12:41.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.053984 12724 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2: Bootstrap starting.
08:12:41.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.061512 12724 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2: Neither blocks nor log segments found. Creating new log.
08:12:41.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.063781 12724 log.cc:826] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2: Log is configured to *not* fsync() on all Append() calls
08:12:41.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.068765 12724 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2: No bootstrap required, opened a new log
08:12:41.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.095747 12724 raft_consensus.cc:357] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } }
08:12:41.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.096674 12724 raft_consensus.cc:383] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:12:41.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.096995 12724 raft_consensus.cc:738] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f5097db0d65045d5926ad111e5e987c2, State: Initialized, Role: FOLLOWER
08:12:41.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.097930 12724 consensus_queue.cc:260] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } }
08:12:41.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.100962 12902 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } } }
08:12:41.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.101966 12902 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [sys.catalog]: This master's current role is: FOLLOWER
08:12:41.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.103004 12724 sys_catalog.cc:564] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [sys.catalog]: configured and running, proceeding with master startup.
08:12:41.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:41.131326 12913 catalog_manager.cc:1560] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:12:41.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:41.131642 12913 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:12:41.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:41.137982 12678 tablet.cc:2378] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:12:41.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.130720 12656 master_runner.cc:186] Error getting master registration for 127.12.89.61:35831: OK, instance_id { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" instance_seqno: 1750839159141576 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:12:41.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.151047 12656 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:12:41.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.408268 12887 raft_consensus.cc:491] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:12:41.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.408818 12887 raft_consensus.cc:513] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } }
08:12:41.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.411242 12887 leader_election.cc:290] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f5097db0d65045d5926ad111e5e987c2 (127.12.89.62:45363), 90567ef9b964421fb03b1f82dab9c962 (127.12.89.61:35831)
08:12:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.412451 12699 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f5097db0d65045d5926ad111e5e987c2" is_pre_election: true
08:12:41.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.413290 12699 raft_consensus.cc:2466] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2f4bf3c4ee5d409bb6d19a3abadc5065 in term 0.
08:12:41.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.414703 12827 leader_election.cc:304] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2f4bf3c4ee5d409bb6d19a3abadc5065, f5097db0d65045d5926ad111e5e987c2; no voters:
08:12:41.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.415550 12887 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:12:41.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.416033 12887 raft_consensus.cc:491] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:12:41.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.416491 12887 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [term 0 FOLLOWER]: Advancing to term 1
08:12:41.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.423238 12887 raft_consensus.cc:513] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } }
08:12:41.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.426148 12887 leader_election.cc:290] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [CANDIDATE]: Term 1 election: Requested vote from peers f5097db0d65045d5926ad111e5e987c2 (127.12.89.62:45363), 90567ef9b964421fb03b1f82dab9c962 (127.12.89.61:35831)
08:12:41.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.426740 12699 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f5097db0d65045d5926ad111e5e987c2"
08:12:41.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.427327 12699 raft_consensus.cc:3058] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [term 0 FOLLOWER]: Advancing to term 1
08:12:41.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.434561 12699 raft_consensus.cc:2466] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2f4bf3c4ee5d409bb6d19a3abadc5065 in term 1.
08:12:41.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.436518 12827 leader_election.cc:304] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2f4bf3c4ee5d409bb6d19a3abadc5065, f5097db0d65045d5926ad111e5e987c2; no voters:
08:12:41.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.437532 12887 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [term 1 FOLLOWER]: Leader election won for term 1
08:12:41.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.443645 12887 raft_consensus.cc:695] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [term 1 LEADER]: Becoming Leader. State: Replica: 2f4bf3c4ee5d409bb6d19a3abadc5065, State: Running, Role: LEADER
08:12:41.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.444800 12887 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } }
08:12:41.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.457533 12915 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2f4bf3c4ee5d409bb6d19a3abadc5065. Latest consensus state: current_term: 1 leader_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } } }
08:12:41.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.458504 12915 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [sys.catalog]: This master's current role is: LEADER
08:12:41.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.465471 12917 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:12:41.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.472585 12917 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:12:41.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.503244 12699 raft_consensus.cc:1273] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [term 1 FOLLOWER]: Refusing update from remote peer 2f4bf3c4ee5d409bb6d19a3abadc5065: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:12:41.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.504976 12915 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:12:41.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.506724 12799 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } attempt: 7
08:12:41.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.529577 12799 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962: Bootstrap starting.
08:12:41.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.543488 12799 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962: Neither blocks nor log segments found. Creating new log.
08:12:41.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.547614 12799 log.cc:826] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962: Log is configured to *not* fsync() on all Append() calls
08:12:41.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.571786 12799 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962: No bootstrap required, opened a new log
08:12:41.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.573129 12918 mvcc.cc:204] Tried to move back new op lower bound from 7171437205454520320 to 7171437205290311680. Current Snapshot: MvccSnapshot[applied={T|T < 7171437205454520320}]
08:12:41.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.577370 12902 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2f4bf3c4ee5d409bb6d19a3abadc5065. Latest consensus state: current_term: 1 leader_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } } }
08:12:41.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.581872 12902 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [sys.catalog]: This master's current role is: FOLLOWER
08:12:41.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.590422 12887 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } } }
08:12:41.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.591122 12887 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [sys.catalog]: This master's current role is: LEADER
08:12:41.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.593861 12917 catalog_manager.cc:1349] Generated new cluster ID: 9ea7ff36ff4444c184893bcdaf81d37c
08:12:41.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.594130 12917 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:12:41.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.626013 12902 sys_catalog.cc:455] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } } }
08:12:41.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.629285 12902 sys_catalog.cc:458] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2 [sys.catalog]: This master's current role is: FOLLOWER
08:12:41.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.644755 12917 catalog_manager.cc:1372] Generated new certificate authority record
08:12:41.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.650892 12917 catalog_manager.cc:1506] Loading token signing keys...
08:12:41.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.654623 12799 raft_consensus.cc:357] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } }
08:12:41.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.655436 12799 raft_consensus.cc:383] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:12:41.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.655718 12799 raft_consensus.cc:738] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 90567ef9b964421fb03b1f82dab9c962, State: Initialized, Role: FOLLOWER
08:12:41.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.656559 12799 consensus_queue.cc:260] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } }
08:12:41.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.676648 12799 sys_catalog.cc:564] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [sys.catalog]: configured and running, proceeding with master startup.
08:12:41.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.676823 12773 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "90567ef9b964421fb03b1f82dab9c962"
08:12:41.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.677557 12773 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [term 0 FOLLOWER]: Advancing to term 1
08:12:41.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.681802 12774 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "90567ef9b964421fb03b1f82dab9c962" is_pre_election: true
08:12:41.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.686995 12773 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2f4bf3c4ee5d409bb6d19a3abadc5065 in term 1.
08:12:41.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.688354 12772 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [term 1 FOLLOWER]: Refusing update from remote peer 2f4bf3c4ee5d409bb6d19a3abadc5065: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:12:41.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.691895 12926 sys_catalog.cc:455] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } } }
08:12:41.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.692700 12926 sys_catalog.cc:458] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [sys.catalog]: This master's current role is: FOLLOWER
08:12:41.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.694389 12887 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [LEADER]: Connected to new peer: Peer: permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:12:41.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.723541 12917 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065: Generated new TSK 0
08:12:41.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.725646 12917 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:12:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.749073 12644 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/wal
08:12:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/data
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/logs
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/data/info.pb
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.1:0
08:12:41.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.89.1
08:12:41.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.1
08:12:41.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:41.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:41.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:41.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:41.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:41.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:12:41.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.780272 12915 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } } }
08:12:41.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.780997 12915 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2f4bf3c4ee5d409bb6d19a3abadc5065 [sys.catalog]: This master's current role is: LEADER
08:12:41.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.780689 12926 sys_catalog.cc:455] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2f4bf3c4ee5d409bb6d19a3abadc5065. Latest consensus state: current_term: 1 leader_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } } }
08:12:41.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.781369 12926 sys_catalog.cc:458] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [sys.catalog]: This master's current role is: FOLLOWER
08:12:41.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.789067 12935 sys_catalog.cc:455] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f5097db0d65045d5926ad111e5e987c2" member_type: VOTER last_known_addr { host: "127.12.89.62" port: 45363 } } peers { permanent_uuid: "90567ef9b964421fb03b1f82dab9c962" member_type: VOTER last_known_addr { host: "127.12.89.61" port: 35831 } } peers { permanent_uuid: "2f4bf3c4ee5d409bb6d19a3abadc5065" member_type: VOTER last_known_addr { host: "127.12.89.60" port: 39467 } } }
08:12:41.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.790122 12935 sys_catalog.cc:458] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962 [sys.catalog]: This master's current role is: FOLLOWER
08:12:41.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.813337 12943 catalog_manager.cc:1261] Loaded cluster ID: 9ea7ff36ff4444c184893bcdaf81d37c
08:12:41.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.813619 12943 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962: loading cluster ID for follower catalog manager: success
08:12:41.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.828210 12943 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962: acquiring CA information for follower catalog manager: success
08:12:41.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:41.833202 12943 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 90567ef9b964421fb03b1f82dab9c962: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:12:42.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:42.137063 12913 catalog_manager.cc:1261] Loaded cluster ID: 9ea7ff36ff4444c184893bcdaf81d37c
08:12:42.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:42.137336 12913 catalog_manager.cc:1554] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2: loading cluster ID for follower catalog manager: success
08:12:42.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:42.152392 12913 catalog_manager.cc:1576] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2: acquiring CA information for follower catalog manager: success
08:12:42.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:42.159137 12913 catalog_manager.cc:1604] T 00000000000000000000000000000000 P f5097db0d65045d5926ad111e5e987c2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:12:42.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:42.339067 12938 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:12:42.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:42.339725 12938 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:12:42.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:42.340488 12938 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:12:42.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:42.399866 12938 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:12:42.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:42.401273 12938 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.89.1
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:42.487044 12938 tablet_server_runner.cc:78] Tablet server non-default flags:
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/data
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/wal
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.1:0
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/data/info.pb
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.1
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:42.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.89.1
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/logs
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:12:42.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:12:42.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:42.488763 12938 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:12:42.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:42.503202 12938 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:12:42.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:42.554229 12950 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:44.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:43.934185 12949 debug-util.cc:398] Leaking SignalData structure 0x7b0800000a80 after lost signal to thread 12938
08:12:44.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:44.578404 12938 thread.cc:640] GCE (cloud detector) Time spent creating pthread: real 2.029s user 0.732s sys 1.069s
08:12:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:44.578794 12938 thread.cc:606] GCE (cloud detector) Time spent starting thread: real 2.029s user 0.732s sys 1.069s
08:12:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:42.555769 12951 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:44.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.586808 12938 server_base.cc:1048] running on GCE node
08:12:44.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:44.587636 12954 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:44.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.588932 12938 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:12:44.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.591336 12938 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:12:44.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.592747 12938 hybrid_clock.cc:648] HybridClock initialized: now 1750839164592651 us; error 78 us; skew 500 ppm
08:12:44.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.593746 12938 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:12:44.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.600669 12938 webserver.cc:469] Webserver started at http://127.12.89.1:43339/ using document root <none> and password file <none>
08:12:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.601810 12938 fs_manager.cc:362] Metadata directory not provided
08:12:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.602062 12938 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:12:44.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.602560 12938 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:12:44.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.608628 12938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/data/instance:
08:12:44.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fb314c9753f454d803fc32e8535c7cf"
08:12:44.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:44 on dist-test-slave-nz51"
08:12:44.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.610035 12938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/wal/instance:
08:12:44.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fb314c9753f454d803fc32e8535c7cf"
08:12:44.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:44 on dist-test-slave-nz51"
08:12:44.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.618832 12938 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.002s
08:12:44.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.625599 12960 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:12:44.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.626678 12938 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
08:12:44.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.627046 12938 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/wal
08:12:44.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6fb314c9753f454d803fc32e8535c7cf"
08:12:44.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:44 on dist-test-slave-nz51"
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.627445 12938 fs_report.cc:389] FS layout report
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/wal
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/wal
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/data/data
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:12:44.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:12:44.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.688725 12938 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:12:44.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.690393 12938 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:12:44.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.690819 12938 kserver.cc:163] Server-wide thread pool size limit: 3276
08:12:44.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.693315 12938 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:12:44.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.697559 12938 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:12:44.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.697752 12938 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:12:44.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.697960 12938 ts_tablet_manager.cc:610] Registered 0 tablets
08:12:44.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.698094 12938 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:12:44.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.827891 12938 rpc_server.cc:307] RPC server started. Bound to: 127.12.89.1:37121
08:12:44.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.828006 13072 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.89.1:37121 every 8 connection(s)
08:12:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.830345 12938 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/data/info.pb
08:12:44.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.837203 12644 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 12938
08:12:44.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.837764 12644 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-0/wal/instance
08:12:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.853207 12644 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/wal
08:12:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/data
08:12:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:12:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:12:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:12:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:44.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/logs
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/data/info.pb
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.2:0
08:12:44.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.89.2
08:12:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.2
08:12:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:12:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.871061 13073 heartbeater.cc:344] Connected to a master server at 127.12.89.60:39467
08:12:44.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.871595 13073 heartbeater.cc:461] Registering TS with master...
08:12:44.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.872902 13073 heartbeater.cc:507] Master 127.12.89.60:39467 requested a full tablet report, sending...
08:12:44.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.874753 13075 heartbeater.cc:344] Connected to a master server at 127.12.89.62:45363
08:12:44.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.875053 13075 heartbeater.cc:461] Registering TS with master...
08:12:44.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.875903 13075 heartbeater.cc:507] Master 127.12.89.62:45363 requested a full tablet report, sending...
08:12:44.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.875883 12843 ts_manager.cc:194] Registered new tserver with Master: 6fb314c9753f454d803fc32e8535c7cf (127.12.89.1:37121)
08:12:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.878531 12689 ts_manager.cc:194] Registered new tserver with Master: 6fb314c9753f454d803fc32e8535c7cf (127.12.89.1:37121)
08:12:44.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.878476 12843 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.89.1:54313
08:12:44.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.879572 13074 heartbeater.cc:344] Connected to a master server at 127.12.89.61:35831
08:12:44.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.879940 13074 heartbeater.cc:461] Registering TS with master...
08:12:44.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.880812 13074 heartbeater.cc:507] Master 127.12.89.61:35831 requested a full tablet report, sending...
08:12:44.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:44.883325 12764 ts_manager.cc:194] Registered new tserver with Master: 6fb314c9753f454d803fc32e8535c7cf (127.12.89.1:37121)
08:12:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:45.154131 13083 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:12:45.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:45.154587 13083 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:12:45.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:45.155036 13083 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:12:45.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:45.187737 13083 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:12:45.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:45.188614 13083 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.89.2
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:45.224068 13083 tablet_server_runner.cc:78] Tablet server non-default flags:
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/data
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/wal
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.2:0
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/data/info.pb
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.2
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.89.2
08:12:45.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/logs
08:12:45.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:45.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:12:45.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:12:45.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:12:45.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:12:45.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:12:45.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:12:45.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:12:45.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:12:45.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:12:45.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:45.225332 13083 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:12:45.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:45.226996 13083 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:12:45.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:45.242477 13090 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:45.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:45.884351 13073 heartbeater.cc:499] Master 127.12.89.60:39467 was elected leader, sending a full tablet report...
08:12:46.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:45.242614 13089 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:46.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:45.243932 13083 server_base.cc:1048] running on GCE node
08:12:46.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:46.356187 13092 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:46.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.358706 13083 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:12:46.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.360863 13083 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:12:46.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.362186 13083 hybrid_clock.cc:648] HybridClock initialized: now 1750839166362144 us; error 55 us; skew 500 ppm
08:12:46.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.362974 13083 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:12:46.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.369364 13083 webserver.cc:469] Webserver started at http://127.12.89.2:37103/ using document root <none> and password file <none>
08:12:46.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.370311 13083 fs_manager.cc:362] Metadata directory not provided
08:12:46.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.370544 13083 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:12:46.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.370949 13083 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:12:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.375624 13083 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/data/instance:
08:12:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa54769dfc74492089ec2da7928904aa"
08:12:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:46 on dist-test-slave-nz51"
08:12:46.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.376803 13083 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/wal/instance:
08:12:46.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa54769dfc74492089ec2da7928904aa"
08:12:46.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:46 on dist-test-slave-nz51"
08:12:46.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.383977 13083 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
08:12:46.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.389809 13101 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:12:46.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.390763 13083 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
08:12:46.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.391072 13083 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/wal
08:12:46.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "aa54769dfc74492089ec2da7928904aa"
08:12:46.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:46 on dist-test-slave-nz51"
08:12:46.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.391387 13083 fs_report.cc:389] FS layout report
08:12:46.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:12:46.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/wal
08:12:46.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/wal
08:12:46.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/data/data
08:12:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:12:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:12:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:12:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:12:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:12:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:12:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:12:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:12:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:12:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:12:46.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:12:46.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.439177 13083 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:12:46.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.440614 13083 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:12:46.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.441107 13083 kserver.cc:163] Server-wide thread pool size limit: 3276
08:12:46.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.443621 13083 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:12:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.448112 13083 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:12:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.448318 13083 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:12:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.448580 13083 ts_tablet_manager.cc:610] Registered 0 tablets
08:12:46.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.448735 13083 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:12:46.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.579440 13083 rpc_server.cc:307] RPC server started. Bound to: 127.12.89.2:35703
08:12:46.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.579552 13213 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.89.2:35703 every 8 connection(s)
08:12:46.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.581959 13083 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/data/info.pb
08:12:46.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.584686 12644 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 13083
08:12:46.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.585314 12644 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-1/wal/instance
08:12:46.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.608523 12644 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/wal
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/data
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/logs
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/data/info.pb
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.3:0
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.89.3
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.3
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:46.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:12:46.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.635393 13214 heartbeater.cc:344] Connected to a master server at 127.12.89.60:39467
08:12:46.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.635900 13214 heartbeater.cc:461] Registering TS with master...
08:12:46.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.637125 13214 heartbeater.cc:507] Master 127.12.89.60:39467 requested a full tablet report, sending...
08:12:46.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.638077 13215 heartbeater.cc:344] Connected to a master server at 127.12.89.61:35831
08:12:46.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.638423 13215 heartbeater.cc:461] Registering TS with master...
08:12:46.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.639322 13215 heartbeater.cc:507] Master 127.12.89.61:35831 requested a full tablet report, sending...
08:12:46.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.639688 12843 ts_manager.cc:194] Registered new tserver with Master: aa54769dfc74492089ec2da7928904aa (127.12.89.2:35703)
08:12:46.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.641323 12843 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.89.2:56433
08:12:46.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.641986 12764 ts_manager.cc:194] Registered new tserver with Master: aa54769dfc74492089ec2da7928904aa (127.12.89.2:35703)
08:12:46.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.645356 13216 heartbeater.cc:344] Connected to a master server at 127.12.89.62:45363
08:12:46.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.645650 13216 heartbeater.cc:461] Registering TS with master...
08:12:46.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.646165 13216 heartbeater.cc:507] Master 127.12.89.62:45363 requested a full tablet report, sending...
08:12:46.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.648185 12689 ts_manager.cc:194] Registered new tserver with Master: aa54769dfc74492089ec2da7928904aa (127.12.89.2:35703)
08:12:46.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:46.931414 13222 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:12:46.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:46.931934 13222 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:12:46.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:46.932456 13222 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:12:46.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:46.963362 13222 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:12:46.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:46.964254 13222 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.89.3
08:12:46.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.998281 13222 tablet_server_runner.cc:78] Tablet server non-default flags:
08:12:46.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:46.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.89.20:45041
08:12:46.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:46.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:46.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/data
08:12:46.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/wal
08:12:46.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:46.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.89.3:0
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/data/info.pb
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.89.3
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.89.62:45363,127.12.89.61:35831,127.12.89.60:39467
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.89.3
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/logs
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:12:46.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:12:47.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:46.999874 13222 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:12:47.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:47.001487 13222 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:12:47.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:47.017311 13231 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:47.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:47.645453 13214 heartbeater.cc:499] Master 127.12.89.60:39467 was elected leader, sending a full tablet report...
08:12:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:47.017688 13233 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:47.017688 13230 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:48.106140 13232 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:12:48.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.106230 13222 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:12:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.110954 13222 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:12:48.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.113545 13222 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:12:48.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.114979 13222 hybrid_clock.cc:648] HybridClock initialized: now 1750839168114935 us; error 61 us; skew 500 ppm
08:12:48.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.115924 13222 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:12:48.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.122393 13222 webserver.cc:469] Webserver started at http://127.12.89.3:45919/ using document root <none> and password file <none>
08:12:48.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.123338 13222 fs_manager.cc:362] Metadata directory not provided
08:12:48.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.123582 13222 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:12:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.124234 13222 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:12:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.128623 13222 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/data/instance:
08:12:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5226abcb481c4d43bc6f19bf20535104"
08:12:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:48 on dist-test-slave-nz51"
08:12:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.129729 13222 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/wal/instance:
08:12:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5226abcb481c4d43bc6f19bf20535104"
08:12:48.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:48 on dist-test-slave-nz51"
08:12:48.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.136683 13222 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
08:12:48.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.142978 13241 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:12:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.144059 13222 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
08:12:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.144337 13222 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/wal
08:12:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5226abcb481c4d43bc6f19bf20535104"
08:12:48.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:48 on dist-test-slave-nz51"
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.144640 13222 fs_report.cc:389] FS layout report
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/wal
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/wal
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/data/data
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:12:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:12:48.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.199784 13222 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:12:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.201200 13222 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:12:48.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.201640 13222 kserver.cc:163] Server-wide thread pool size limit: 3276
08:12:48.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.204128 13222 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:12:48.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.208371 13222 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:12:48.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.208590 13222 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:12:48.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.208882 13222 ts_tablet_manager.cc:610] Registered 0 tablets
08:12:48.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.209062 13222 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:12:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.336308 13222 rpc_server.cc:307] RPC server started. Bound to: 127.12.89.3:41011
08:12:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.336419 13353 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.89.3:41011 every 8 connection(s)
08:12:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.338794 13222 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/data/info.pb
08:12:48.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.345335 12644 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 13222
08:12:48.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.345829 12644 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster7072104984696305593/ts-2/wal/instance
08:12:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.389528 13354 heartbeater.cc:344] Connected to a master server at 127.12.89.60:39467
08:12:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.390059 13354 heartbeater.cc:461] Registering TS with master...
08:12:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.390870 13355 heartbeater.cc:344] Connected to a master server at 127.12.89.61:35831
08:12:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.391196 13355 heartbeater.cc:461] Registering TS with master...
08:12:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.391337 13354 heartbeater.cc:507] Master 127.12.89.60:39467 requested a full tablet report, sending...
08:12:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.392311 13355 heartbeater.cc:507] Master 127.12.89.61:35831 requested a full tablet report, sending...
08:12:48.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.394443 12842 ts_manager.cc:194] Registered new tserver with Master: 5226abcb481c4d43bc6f19bf20535104 (127.12.89.3:41011)
08:12:48.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.395292 12764 ts_manager.cc:194] Registered new tserver with Master: 5226abcb481c4d43bc6f19bf20535104 (127.12.89.3:41011)
08:12:48.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.396212 12842 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.12.89.3:42079
08:12:48.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.396713 13356 heartbeater.cc:344] Connected to a master server at 127.12.89.62:45363
08:12:48.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.396997 13356 heartbeater.cc:461] Registering TS with master...
08:12:48.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.399272 13356 heartbeater.cc:507] Master 127.12.89.62:45363 requested a full tablet report, sending...
08:12:48.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.401608 12689 ts_manager.cc:194] Registered new tserver with Master: 5226abcb481c4d43bc6f19bf20535104 (127.12.89.3:41011)
08:12:48.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.403581 12644 external_mini_cluster.cc:934] 3 TS(s) registered with all masters
08:12:48.414 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:12:48.414 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:12:48.415 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.12.89.62"
port: 45363
}
bound_http_address {
host: "127.12.89.62"
port: 39263
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.12.89.61"
port: 35831
}
bound_http_address {
host: "127.12.89.61"
port: 34809
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.12.89.60"
port: 39467
}
bound_http_address {
host: "127.12.89.60"
port: 37393
}
}
}
08:12:48.415 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:12:48.417 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.12.89.1"
port: 37121
}
bound_http_address {
host: "127.12.89.1"
port: 43339
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.12.89.2"
port: 35703
}
bound_http_address {
host: "127.12.89.2"
port: 37103
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.12.89.3"
port: 41011
}
bound_http_address {
host: "127.12.89.3"
port: 45919
}
}
}
08:12:48.417 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:12:48.417 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster651919651313464243/krb5kdc/krb5cc
08:12:48.418 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:12:48.419 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskzFzfv3/test-tmp
08:12:48.419 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:12:48.419 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskzFzfv3/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:12:48.421 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150"
extra_master_flags: "--authn_token_validity_seconds=10"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:12:48.735 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:12:48.735 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:12:48.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.738440 13365 mini_oidc.cc:150] Starting JWKS server
08:12:48.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.741777 13365 webserver.cc:469] Webserver started at http://127.0.0.1:44789/ using document root <none> and password file <none>
08:12:48.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.742180 13365 mini_oidc.cc:186] Starting OIDC Discovery server
08:12:48.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:48.743716 13365 webserver.cc:469] Webserver started at http://127.0.0.1:44827/ using document root <none> and password file <none>
08:12:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:12:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/principal' for realm 'KRBTEST.COM',
08:12:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:12:48.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:48 dist-test-slave-nz51 krb5kdc[13376](info): setting up network...
08:12:48.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:12:48.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:48 dist-test-slave-nz51 krb5kdc[13376](info): set up 2 sockets
08:12:48.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:48 dist-test-slave-nz51 krb5kdc[13376](info): commencing operation
08:12:48.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:12:49.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:49.399919 13354 heartbeater.cc:499] Master 127.12.89.60:39467 was elected leader, sending a full tablet report...
08:12:50.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:50.653719 13365 mini_kdc.cc:121] Time spent starting KDC: real 1.874s user 0.003s sys 0.016s
08:12:50.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:12:50.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:12:50.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:12:50.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:12:50.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:12:50.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:12:50.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:12:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:12:50.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:12:50.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:12:50.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/test-user.keytab.
08:12:50.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/test-user.keytab.
08:12:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:50 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839170, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:12:50.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:12:50.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T08:12:50Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:12:50.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T08:12:50Z Disabled control of system clock
08:12:50.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.13.126@KRBTEST.COM; defaulting to no policy
08:12:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:12:50.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.13.126@KRBTEST.COM" created.
08:12:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:12:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.13.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.126.keytab.
08:12:50.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.13.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.126.keytab.
08:12:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:50.825207 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:12:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data
08:12:50.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/logs
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.126:37371
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.126
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.126.keytab
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.126
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:12:50.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:12:51.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.124125 13391 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:12:51.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.124729 13391 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:12:51.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.125340 13391 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:12:51.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.157344 13391 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:12:51.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.157632 13391 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:12:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.157917 13391 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:12:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.158114 13391 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:12:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.158299 13391 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:12:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.158484 13391 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:12:51.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.158622 13391 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:51.195827 13391 master_runner.cc:386] Master server non-default flags:
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.126.keytab
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.126
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.126:37371
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.126
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/logs
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:12:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:12:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:51.197105 13391 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:12:51.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:51.198149 13391 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:12:52.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.212642 13397 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:52.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.212687 13400 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:52.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:51.212675 13398 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:52.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:51.213979 13391 server_base.cc:1048] running on GCE node
08:12:52.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.334323 13391 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:12:52.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.337313 13391 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:12:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.338758 13391 hybrid_clock.cc:648] HybridClock initialized: now 1750839172338704 us; error 73 us; skew 500 ppm
08:12:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.339583 13391 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:12:52.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:52 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839172, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:12:52.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.349614 13391 init.cc:377] Logged in from keytab as kudu/127.13.13.126@KRBTEST.COM (short username kudu)
08:12:52.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.354858 13391 webserver.cc:469] Webserver started at http://127.13.13.126:42523/ using document root <none> and password file <none>
08:12:52.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.355723 13391 fs_manager.cc:362] Metadata directory not provided
08:12:52.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.355912 13391 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:12:52.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.356299 13391 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:12:52.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.360618 13391 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/instance:
08:12:52.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "268581e0456549218cd7fff0cb991913"
08:12:52.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:52 on dist-test-slave-nz51"
08:12:52.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.361778 13391 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal/instance:
08:12:52.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "268581e0456549218cd7fff0cb991913"
08:12:52.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:52 on dist-test-slave-nz51"
08:12:52.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.368866 13391 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
08:12:52.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.374310 13409 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:12:52.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.375387 13391 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
08:12:52.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.375661 13391 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:12:52.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "268581e0456549218cd7fff0cb991913"
08:12:52.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:52 on dist-test-slave-nz51"
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.375949 13391 fs_report.cc:389] FS layout report
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/data
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:12:52.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:12:52.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.432307 13391 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:12:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.437791 13391 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:12:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.438256 13391 kserver.cc:163] Server-wide thread pool size limit: 3276
08:12:52.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.507966 13391 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.126:37371
08:12:52.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.508069 13460 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.126:37371 every 8 connection(s)
08:12:52.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.510696 13391 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:12:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.515703 13461 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:12:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.520886 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 13391
08:12:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.521324 13365 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal/instance
08:12:52.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.526505 13461 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } has no permanent_uuid. Determining permanent_uuid...
08:12:52.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.13.125@KRBTEST.COM; defaulting to no policy
08:12:52.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:12:52.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.13.125@KRBTEST.COM" created.
08:12:52.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:52 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839172, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.126@KRBTEST.COM for kudu/127.13.13.126@KRBTEST.COM
08:12:52.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:12:52.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.13.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.125.keytab.
08:12:52.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.13.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.125.keytab.
08:12:52.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.562001 13461 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } has no permanent_uuid. Determining permanent_uuid...
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.563321 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/logs
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:12:52.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.125:37877
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.125
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.125.keytab
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.125
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:12:52.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.564548 13413 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.13.125:37877: connect: Connection refused (error 111)
08:12:52.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.568375 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.125:37877: Network error: Client connection negotiation failed: client connection to 127.13.13.125:37877: connect: Connection refused (error 111)
08:12:52.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.617910 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } attempt: 1
08:12:52.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.621578 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.125:37877: Network error: Client connection negotiation failed: client connection to 127.13.13.125:37877: connect: Connection refused (error 111)
08:12:52.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.690047 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } attempt: 2
08:12:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.693675 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.125:37877: Network error: Client connection negotiation failed: client connection to 127.13.13.125:37877: connect: Connection refused (error 111)
08:12:52.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.785116 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } attempt: 3
08:12:52.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.788796 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.125:37877: Network error: Client connection negotiation failed: client connection to 127.13.13.125:37877: connect: Connection refused (error 111)
08:12:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.865949 13468 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:12:52.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.866466 13468 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:12:52.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.867048 13468 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:12:52.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.898304 13468 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:12:52.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.898537 13468 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:12:52.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.898801 13468 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:12:52.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.899008 13468 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:12:52.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.899199 13468 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:12:52.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.899384 13468 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:12:52.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.899523 13468 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:12:52.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.932236 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } attempt: 4
08:12:52.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.936345 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.125:37877: Network error: Client connection negotiation failed: client connection to 127.13.13.125:37877: connect: Connection refused (error 111)
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.939630 13468 master_runner.cc:386] Master server non-default flags:
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.125.keytab
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.125
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.125:37877
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.125
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/logs
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:12:52.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:12:52.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.940935 13468 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:12:52.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.941993 13468 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:12:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.956305 13477 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:53.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:53.235800 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } attempt: 5
08:12:53.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:53.239743 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.125:37877: Network error: Client connection negotiation failed: client connection to 127.13.13.125:37877: connect: Connection refused (error 111)
08:12:53.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:53.787093 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } attempt: 6
08:12:53.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:53.793231 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.125:37877: Network error: Client connection negotiation failed: client connection to 127.13.13.125:37877: connect: Connection refused (error 111)
08:12:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:52.958256 13468 server_base.cc:1048] running on GCE node
08:12:54.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.957027 13479 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:54.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:52.956521 13476 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:54.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.079222 13468 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:12:54.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.082340 13468 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:12:54.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.083771 13468 hybrid_clock.cc:648] HybridClock initialized: now 1750839174083728 us; error 64 us; skew 500 ppm
08:12:54.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.084586 13468 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:12:54.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:54 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839174, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:12:54.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.090165 13468 init.cc:377] Logged in from keytab as kudu/127.13.13.125@KRBTEST.COM (short username kudu)
08:12:54.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.095017 13468 webserver.cc:469] Webserver started at http://127.13.13.125:32937/ using document root <none> and password file <none>
08:12:54.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.095916 13468 fs_manager.cc:362] Metadata directory not provided
08:12:54.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.096104 13468 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:12:54.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.096524 13468 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:12:54.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.100826 13468 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/instance:
08:12:54.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd4c83ae72e94b369d028ea29fc7e62c"
08:12:54.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:54 on dist-test-slave-nz51"
08:12:54.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.101895 13468 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal/instance:
08:12:54.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd4c83ae72e94b369d028ea29fc7e62c"
08:12:54.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:54 on dist-test-slave-nz51"
08:12:54.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.109179 13468 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
08:12:54.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.114701 13489 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:12:54.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.115701 13468 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.116003 13468 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd4c83ae72e94b369d028ea29fc7e62c"
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:54 on dist-test-slave-nz51"
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.116321 13468 fs_report.cc:389] FS layout report
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/data
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:12:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:12:54.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:12:54.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:12:54.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:12:54.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:12:54.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:12:54.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:12:54.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:12:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.184885 13468 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:12:54.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.190111 13468 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:12:54.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.190505 13468 kserver.cc:163] Server-wide thread pool size limit: 3276
08:12:54.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.259337 13468 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.125:37877
08:12:54.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.259418 13541 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.125:37877 every 8 connection(s)
08:12:54.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.261924 13468 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:12:54.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.262687 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 13468
08:12:54.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.263084 13365 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal/instance
08:12:54.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.267661 13542 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:12:54.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.13.124@KRBTEST.COM; defaulting to no policy
08:12:54.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:12:54.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.13.124@KRBTEST.COM" created.
08:12:54.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.282335 13542 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } has no permanent_uuid. Determining permanent_uuid...
08:12:54.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:12:54.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.13.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.124.keytab.
08:12:54.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.13.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.124.keytab.
08:12:54.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:54 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839174, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.125@KRBTEST.COM for kudu/127.13.13.126@KRBTEST.COM
08:12:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.303448 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:12:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data
08:12:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:12:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:12:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:12:54.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/logs
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.124:46557
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.124
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.124.keytab
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.124
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:12:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:12:54.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.314101 13542 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } has no permanent_uuid. Determining permanent_uuid...
08:12:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:54 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839174, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.125@KRBTEST.COM for kudu/127.13.13.125@KRBTEST.COM
08:12:54.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.337906 13542 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } has no permanent_uuid. Determining permanent_uuid...
08:12:54.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.340143 13490 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:54.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.343936 13542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:54.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.393379 13542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 1
08:12:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.397253 13542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.466553 13542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 2
08:12:54.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.476557 13542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:54.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.568023 13542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 3
08:12:54.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.572377 13542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:54.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.649525 13548 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:12:54.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.650137 13548 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:12:54.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.650769 13548 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:12:54.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.683882 13548 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:12:54.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.684144 13548 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:12:54.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.684504 13548 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:12:54.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.684779 13548 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:12:54.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.685041 13548 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:12:54.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.685293 13548 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:12:54.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.685514 13548 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:12:54.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.715768 13542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 4
08:12:54.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.720373 13542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.724771 13548 master_runner.cc:386] Master server non-default flags:
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.124.keytab
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.124
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.124:46557
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:12:54.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.124
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/logs
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:12:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:12:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:12:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.726459 13548 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:12:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.727690 13548 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:12:54.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.742296 13557 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.853788 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } attempt: 7
08:12:54.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:54 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839172, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.126@KRBTEST.COM for kudu/127.13.13.125@KRBTEST.COM
08:12:54.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.877957 13461 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } has no permanent_uuid. Determining permanent_uuid...
08:12:54.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.883767 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:54.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.942412 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 1
08:12:54.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.946345 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:55.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.019749 13542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 5
08:12:55.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:55.024405 13542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:55.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.027866 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 2
08:12:55.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:55.031729 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:55.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.117229 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 3
08:12:55.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:55.121042 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:55.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.261521 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 4
08:12:55.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:55.266273 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:55.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:55.500329 13069 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5620 after lost signal to thread 12945
08:12:55.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:55.501178 13069 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5520 after lost signal to thread 13072
08:12:55.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.549713 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 5
08:12:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:55.554687 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:55.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.571797 13542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 6
08:12:55.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:55.580415 13542 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.742334 13556 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:54.742340 13559 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:12:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:54.744889 13548 server_base.cc:1048] running on GCE node
08:12:55.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.965850 13548 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:12:55.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.969563 13548 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:12:55.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.971140 13548 hybrid_clock.cc:648] HybridClock initialized: now 1750839175970944 us; error 93 us; skew 500 ppm
08:12:55.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.972110 13548 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:12:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:55 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839175, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:12:55.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.978895 13548 init.cc:377] Logged in from keytab as kudu/127.13.13.124@KRBTEST.COM (short username kudu)
08:12:55.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.989260 13548 webserver.cc:469] Webserver started at http://127.13.13.124:36079/ using document root <none> and password file <none>
08:12:55.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.990567 13548 fs_manager.cc:362] Metadata directory not provided
08:12:55.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.990846 13548 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:12:55.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.991403 13548 server_base.cc:896] This appears to be a new deployment of Kudu; creating new FS layout
08:12:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.997372 13548 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/instance:
08:12:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f97e33e85db48b3a34d544fa3495dd9"
08:12:55.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:55 on dist-test-slave-nz51"
08:12:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:55.998528 13548 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal/instance:
08:12:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f97e33e85db48b3a34d544fa3495dd9"
08:12:55.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:55 on dist-test-slave-nz51"
08:12:56.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.006103 13548 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
08:12:56.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.012058 13574 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:12:56.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.013166 13548 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
08:12:56.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.013612 13548 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f97e33e85db48b3a34d544fa3495dd9"
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:55 on dist-test-slave-nz51"
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.013957 13548 fs_report.cc:389] FS layout report
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/data
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:12:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:12:56.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.076574 13548 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:12:56.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.084555 13548 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:12:56.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.085189 13548 kserver.cc:163] Server-wide thread pool size limit: 3276
08:12:56.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.107138 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 6
08:12:56.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:56.115315 13461 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:12:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.176090 13548 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.124:46557
08:12:56.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.176271 13628 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.124:46557 every 8 connection(s)
08:12:56.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.179853 13548 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:12:56.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.186163 13629 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:12:56.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.189760 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 13548
08:12:56.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.190217 13365 external_mini_cluster.cc:1427] Reading /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal/instance
08:12:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.214186 13629 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } has no permanent_uuid. Determining permanent_uuid...
08:12:56.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:56 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839170, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.13.124@KRBTEST.COM
08:12:56.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:56 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839175, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.124@KRBTEST.COM for kudu/127.13.13.126@KRBTEST.COM
08:12:56.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.255075 13629 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } has no permanent_uuid. Determining permanent_uuid...
08:12:56.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:56 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839175, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.124@KRBTEST.COM for kudu/127.13.13.125@KRBTEST.COM
08:12:56.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:56 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839170, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.13.125@KRBTEST.COM
08:12:56.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.293588 13629 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } has no permanent_uuid. Determining permanent_uuid...
08:12:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:56 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839175, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.124@KRBTEST.COM for kudu/127.13.13.124@KRBTEST.COM
08:12:56.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:56 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839170, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.13.126@KRBTEST.COM
08:12:56.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.322590 13629 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Bootstrap starting.
08:12:56.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.330104 13629 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Neither blocks nor log segments found. Creating new log.
08:12:56.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.330319 13365 external_mini_cluster.cc:934] 0 TS(s) registered with all masters
08:12:56.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.332412 13629 log.cc:826] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Log is configured to *not* fsync() on all Append() calls
08:12:56.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.338097 13629 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: No bootstrap required, opened a new log
08:12:56.343 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:12:56.343 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:12:56.344 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:12:56.344 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:12:56.345 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.13.13.126"
port: 37371
}
bound_http_address {
host: "127.13.13.126"
port: 42523
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.13.13.125"
port: 37877
}
bound_http_address {
host: "127.13.13.125"
port: 32937
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.13.13.124"
port: 46557
}
bound_http_address {
host: "127.13.13.124"
port: 36079
}
}
}
08:12:56.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:12:56.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:12:56.346 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:12:56.347 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:12:56.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.361505 13629 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:12:56.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.362345 13629 raft_consensus.cc:383] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:12:56.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.362733 13629 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3f97e33e85db48b3a34d544fa3495dd9, State: Initialized, Role: FOLLOWER
08:12:56.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.363566 13629 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:12:56.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.366295 13639 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:12:56.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.367256 13639 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: This master's current role is: FOLLOWER
08:12:56.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.368338 13629 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: configured and running, proceeding with master startup.
08:12:56.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:56 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839176, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:12:56.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:12:56.391 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:12:56.392 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc
08:12:56.393 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:12:56.393 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc
08:12:56.395 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:12:56.398 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
08:12:56.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:56.406863 13651 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:12:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:56.407343 13651 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:12:56.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.420840 13639 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:12:56.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.421375 13639 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:12:56.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.393182 13548 master_runner.cc:186] Error getting master registration for 127.13.13.126:37371: OK, instance_id { permanent_uuid: "268581e0456549218cd7fff0cb991913" instance_seqno: 1750839172489830 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:12:56.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.423038 13548 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:12:56.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.423837 13639 leader_election.cc:290] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371), dd4c83ae72e94b369d028ea29fc7e62c (127.13.13.125:37877)
08:12:56.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:56.591594 13580 tablet.cc:2378] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:12:56.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.640985 13542 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 7
08:12:56.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:56 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839174, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.125@KRBTEST.COM for kudu/127.13.13.124@KRBTEST.COM
08:12:56.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.703819 13542 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Bootstrap starting.
08:12:56.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.711596 13542 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Neither blocks nor log segments found. Creating new log.
08:12:56.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.714255 13542 log.cc:826] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Log is configured to *not* fsync() on all Append() calls
08:12:56.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.720206 13542 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: No bootstrap required, opened a new log
08:12:56.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.747661 13542 raft_consensus.cc:357] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:12:56.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.748522 13542 raft_consensus.cc:383] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:12:56.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.749133 13542 raft_consensus.cc:738] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd4c83ae72e94b369d028ea29fc7e62c, State: Initialized, Role: FOLLOWER
08:12:56.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.750080 13542 consensus_queue.cc:260] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:12:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.758713 13655 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:12:56.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.759516 13655 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: FOLLOWER
08:12:56.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.763324 13542 sys_catalog.cc:564] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: configured and running, proceeding with master startup.
08:12:56.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.763509 13517 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3f97e33e85db48b3a34d544fa3495dd9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" is_pre_election: true
08:12:56.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.764533 13517 raft_consensus.cc:2466] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3f97e33e85db48b3a34d544fa3495dd9 in term 0.
08:12:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.766208 13578 leader_election.cc:304] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3f97e33e85db48b3a34d544fa3495dd9, dd4c83ae72e94b369d028ea29fc7e62c; no voters:
08:12:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.767119 13639 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:12:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.767443 13639 raft_consensus.cc:491] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:12:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.767740 13639 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 0 FOLLOWER]: Advancing to term 1
08:12:56.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.775192 13639 raft_consensus.cc:513] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:12:56.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.780450 13517 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3f97e33e85db48b3a34d544fa3495dd9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dd4c83ae72e94b369d028ea29fc7e62c"
08:12:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.781184 13517 raft_consensus.cc:3058] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 0 FOLLOWER]: Advancing to term 1
08:12:56.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.781657 13639 leader_election.cc:290] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [CANDIDATE]: Term 1 election: Requested vote from peers 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371), dd4c83ae72e94b369d028ea29fc7e62c (127.13.13.125:37877)
08:12:56.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.797070 13517 raft_consensus.cc:2466] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3f97e33e85db48b3a34d544fa3495dd9 in term 1.
08:12:56.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.798724 13578 leader_election.cc:304] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3f97e33e85db48b3a34d544fa3495dd9, dd4c83ae72e94b369d028ea29fc7e62c; no voters:
08:12:56.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.799768 13639 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 1 FOLLOWER]: Leader election won for term 1
08:12:56.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.810799 13639 raft_consensus.cc:695] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 1 LEADER]: Becoming Leader. State: Replica: 3f97e33e85db48b3a34d544fa3495dd9, State: Running, Role: LEADER
08:12:56.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.811956 13639 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:12:56.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.847566 13658 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3f97e33e85db48b3a34d544fa3495dd9. Latest consensus state: current_term: 1 leader_uuid: "3f97e33e85db48b3a34d544fa3495dd9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:12:56.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.850325 13658 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: This master's current role is: LEADER
08:12:56.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.862490 13664 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:12:56.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.869364 13664 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:12:56.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.912308 13517 raft_consensus.cc:1273] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Refusing update from remote peer 3f97e33e85db48b3a34d544fa3495dd9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:12:56.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.914225 13658 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:12:56.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:56.916771 13669 catalog_manager.cc:1560] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:12:56.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:56.917088 13669 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:12:56.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:56.950672 13495 tablet.cc:2378] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:12:56.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.897624 13468 master_runner.cc:186] Error getting master registration for 127.13.13.126:37371: OK, instance_id { permanent_uuid: "268581e0456549218cd7fff0cb991913" instance_seqno: 1750839172489830 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:12:56.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.982641 13468 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:12:56.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:56.997023 13670 mvcc.cc:204] Tried to move back new op lower bound from 7171437268600295424 to 7171437268233080832. Current Snapshot: MvccSnapshot[applied={T|T < 7171437268600295424}]
08:12:57.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.011755 13673 mvcc.cc:204] Tried to move back new op lower bound from 7171437268600295424 to 7171437268233080832. Current Snapshot: MvccSnapshot[applied={T|T < 7171437268600295424}]
08:12:57.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.018980 13676 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3f97e33e85db48b3a34d544fa3495dd9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:12:57.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.022328 13676 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: FOLLOWER
08:12:57.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.014755 13655 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3f97e33e85db48b3a34d544fa3495dd9. Latest consensus state: current_term: 1 leader_uuid: "3f97e33e85db48b3a34d544fa3495dd9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:12:57.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.026182 13639 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3f97e33e85db48b3a34d544fa3495dd9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:12:57.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.026860 13639 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: This master's current role is: LEADER
08:12:57.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.024752 13655 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: FOLLOWER
08:12:57.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.036172 13664 catalog_manager.cc:1349] Generated new cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:12:57.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.036481 13664 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:12:57.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.148733 13461 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } attempt: 7
08:12:57.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.155871 13664 catalog_manager.cc:1372] Generated new certificate authority record
08:12:57.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.163121 13664 catalog_manager.cc:1506] Loading token signing keys...
08:12:57.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:12:57 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839172, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.126@KRBTEST.COM for kudu/127.13.13.124@KRBTEST.COM
08:12:57.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.234391 13664 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Generated new TSK 0
08:12:57.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.236487 13461 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Bootstrap starting.
08:12:57.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.239287 13664 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:12:57.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.259552 13461 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Neither blocks nor log segments found. Creating new log.
08:12:57.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.268100 13461 log.cc:826] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Log is configured to *not* fsync() on all Append() calls
08:12:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.289041 13461 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: No bootstrap required, opened a new log
08:12:57.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.356982 13461 raft_consensus.cc:357] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:12:57.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.385216 13461 raft_consensus.cc:383] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:12:57.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.385785 13461 raft_consensus.cc:738] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 268581e0456549218cd7fff0cb991913, State: Initialized, Role: FOLLOWER
08:12:57.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.386992 13461 consensus_queue.cc:260] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:12:57.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.402289 13680 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:12:57.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.406085 13461 sys_catalog.cc:564] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: configured and running, proceeding with master startup.
08:12:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.406292 13435 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3f97e33e85db48b3a34d544fa3495dd9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "268581e0456549218cd7fff0cb991913"
08:12:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.406973 13436 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3f97e33e85db48b3a34d544fa3495dd9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "268581e0456549218cd7fff0cb991913" is_pre_election: true
08:12:57.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.408104 13434 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 0 FOLLOWER]: Advancing to term 1
08:12:57.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.413712 13680 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:12:57.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.418474 13434 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 1 FOLLOWER]: Refusing update from remote peer 3f97e33e85db48b3a34d544fa3495dd9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:12:57.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.421761 13658 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:12:57.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:57.450384 13415 tablet.cc:2378] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:12:57.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.514288 13680 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3f97e33e85db48b3a34d544fa3495dd9. Latest consensus state: current_term: 1 leader_uuid: "3f97e33e85db48b3a34d544fa3495dd9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:12:57.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.515259 13680 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:12:57.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.516587 13687 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3f97e33e85db48b3a34d544fa3495dd9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:12:57.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.521543 13687 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:12:57.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.563446 13696 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:12:57.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.922845 13669 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:12:57.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.923141 13669 catalog_manager.cc:1554] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: loading cluster ID for follower catalog manager: success
08:12:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.930469 13669 catalog_manager.cc:1576] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: acquiring CA information for follower catalog manager: success
08:12:57.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:57.936144 13669 catalog_manager.cc:1604] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:12:58.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:58.003980 13658 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3f97e33e85db48b3a34d544fa3495dd9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:12:58.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:58.004760 13658 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: This master's current role is: LEADER
08:12:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:58.571892 13696 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:12:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:58.572391 13696 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: loading cluster ID for follower catalog manager: success
08:12:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:58.591457 13696 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: acquiring CA information for follower catalog manager: success
08:12:58.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:58.597219 13696 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:12:59.399 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.13.125:37877
08:12:59.399 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:12:59.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:59.400486 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 13468
08:12:59.423 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:12:59.423 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.13.126:37371
08:12:59.423 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:12:59.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:59.424015 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 13391
08:12:59.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:12:59.448 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.13.124:46557
08:12:59.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:12:59.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:59.448868 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 13548
08:12:59.474 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:12:59.474 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.13.125:37877
08:12:59.475 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:59.477025 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/logs
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:59.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.125:37877
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.125
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32937
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.125.keytab
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.125
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:12:59.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:12:59.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.765161 13705 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:12:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.765810 13705 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:12:59.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.766476 13705 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:12:59.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.798583 13705 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:12:59.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.798859 13705 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:12:59.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.799239 13705 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:12:59.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.799510 13705 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:12:59.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.799777 13705 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:12:59.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.800036 13705 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:12:59.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.800241 13705 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:12:59.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:59.838233 13705 master_runner.cc:386] Master server non-default flags:
08:12:59.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:12:59.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:12:59.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:12:59.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:12:59.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data
08:12:59.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:12:59.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:12:59.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:12:59.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.125.keytab
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.125
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.125:37877
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.125
08:12:59.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32937
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/logs
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:12:59.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:59.839838 13705 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:12:59.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:59.841010 13705 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:12:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.854626 13711 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.855547 13714 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:12:59.856452 13705 server_base.cc:1048] running on GCE node
08:13:00.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:12:59.855822 13712 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:00.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:00.973634 13705 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:13:00.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:00.976140 13705 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:13:00.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:00.977511 13705 hybrid_clock.cc:648] HybridClock initialized: now 1750839180977473 us; error 47 us; skew 500 ppm
08:13:00.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:00.978307 13705 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:13:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:00 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839180, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:13:00.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:00.987232 13705 init.cc:377] Logged in from keytab as kudu/127.13.13.125@KRBTEST.COM (short username kudu)
08:13:00.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:00.992472 13705 webserver.cc:469] Webserver started at http://127.13.13.125:32937/ using document root <none> and password file <none>
08:13:00.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:00.993352 13705 fs_manager.cc:362] Metadata directory not provided
08:13:00.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:00.993595 13705 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:13:01.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.001204 13705 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.001s
08:13:01.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.005816 13722 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:13:01.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.006919 13705 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
08:13:01.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.007274 13705 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:01.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd4c83ae72e94b369d028ea29fc7e62c"
08:13:01.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:54 on dist-test-slave-nz51"
08:13:01.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.009138 13705 fs_report.cc:389] FS layout report
08:13:01.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:13:01.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/data
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:13:01.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:13:01.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.067180 13705 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:13:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.073550 13705 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:13:01.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.074133 13705 kserver.cc:163] Server-wide thread pool size limit: 3276
08:13:01.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.143388 13705 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.125:37877
08:13:01.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.143464 13773 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.125:37877 every 8 connection(s)
08:13:01.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.146338 13705 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:13:01.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.156196 13774 sys_catalog.cc:263] Verifying existing consensus state
08:13:01.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.156756 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 13705
08:13:01.157 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:01.157 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.13.126:37371
08:13:01.157 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.159222 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/logs
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:13:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.126:37371
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.126
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42523
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.126.keytab
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.126
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:01.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:13:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.163533 13774 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Bootstrap starting.
08:13:01.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.208853 13774 log.cc:826] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Log is configured to *not* fsync() on all Append() calls
08:13:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.224481 13774 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:13:01.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.225323 13774 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Bootstrap complete.
08:13:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.247812 13774 raft_consensus.cc:357] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:01.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.250309 13774 raft_consensus.cc:738] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd4c83ae72e94b369d028ea29fc7e62c, State: Initialized, Role: FOLLOWER
08:13:01.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.251125 13774 consensus_queue.cc:260] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:01.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.253803 13778 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:01.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.254642 13778 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: FOLLOWER
08:13:01.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.255542 13774 sys_catalog.cc:564] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: configured and running, proceeding with master startup.
08:13:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.282711 13789 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:13:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.282974 13789 catalog_manager.cc:1554] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: loading cluster ID for follower catalog manager: success
08:13:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.291136 13789 catalog_manager.cc:1576] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: acquiring CA information for follower catalog manager: success
08:13:01.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.296550 13789 catalog_manager.cc:1604] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:13:01.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.286834 13705 master_runner.cc:186] Error getting master registration for 127.13.13.126:37371: Network error: Client connection negotiation failed: client connection to 127.13.13.126:37371: connect: Connection refused (error 111),
08:13:01.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.314877 13705 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:13:01.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.497473 13776 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:13:01.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.498250 13776 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:13:01.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.499293 13776 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:13:01.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.532083 13776 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:13:01.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.532361 13776 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:13:01.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.532732 13776 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:13:01.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.533007 13776 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:13:01.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.533281 13776 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:13:01.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.533560 13776 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:13:01.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.533830 13776 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:13:01.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.573578 13776 master_runner.cc:386] Master server non-default flags:
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.126.keytab
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.126
08:13:01.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.126:37371
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.126
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42523
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/logs
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:13:01.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.575227 13776 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:13:01.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:01.576406 13776 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:13:01.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.592972 13797 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:03.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.013286 13804 raft_consensus.cc:491] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:13:03.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.015799 13804 raft_consensus.cc:513] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:03.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.038374 13804 leader_election.cc:290] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371), 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557)
08:13:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:03.040278 13723 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:13:03.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:03.052115 13723 leader_election.cc:336] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557): Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:13:03.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:03.055310 13725 leader_election.cc:336] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371): Network error: Client connection negotiation failed: client connection to 127.13.13.126:37371: connect: Connection refused (error 111)
08:13:03.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.055821 13725 leader_election.cc:304] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dd4c83ae72e94b369d028ea29fc7e62c; no voters: 268581e0456549218cd7fff0cb991913, 3f97e33e85db48b3a34d544fa3495dd9
08:13:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.056990 13804 raft_consensus.cc:2747] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:13:03.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:02.991310 13796 debug-util.cc:398] Leaking SignalData structure 0x7b0800018780 after lost signal to thread 13776
08:13:03.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:03.427536 13799 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1832 milliseconds
08:13:03.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:01.595675 13798 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:03.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:03.426596 13776 thread.cc:640] OpenStack (cloud detector) Time spent creating pthread: real 1.831s user 0.625s sys 1.099s
08:13:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:03.428877 13776 thread.cc:606] OpenStack (cloud detector) Time spent starting thread: real 1.834s user 0.625s sys 1.100s
08:13:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:03.429030 13800 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:03.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.429580 13776 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:13:03.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.432900 13776 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:13:03.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.435828 13776 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:13:03.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.437314 13776 hybrid_clock.cc:648] HybridClock initialized: now 1750839183437231 us; error 91 us; skew 500 ppm
08:13:03.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.438506 13776 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:13:03.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:03 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839183, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:13:03.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.445225 13776 init.cc:377] Logged in from keytab as kudu/127.13.13.126@KRBTEST.COM (short username kudu)
08:13:03.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.457852 13776 webserver.cc:469] Webserver started at http://127.13.13.126:42523/ using document root <none> and password file <none>
08:13:03.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.459172 13776 fs_manager.cc:362] Metadata directory not provided
08:13:03.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.459463 13776 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:13:03.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.479796 13776 fs_manager.cc:714] Time spent opening directory manager: real 0.016s user 0.005s sys 0.009s
08:13:03.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.486690 13813 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:13:03.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.488106 13776 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.003s
08:13:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.488448 13776 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "268581e0456549218cd7fff0cb991913"
08:13:03.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:52 on dist-test-slave-nz51"
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.490751 13776 fs_report.cc:389] FS layout report
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/data
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:13:03.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:13:03.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.547729 13776 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:13:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.553815 13776 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:13:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.554235 13776 kserver.cc:163] Server-wide thread pool size limit: 3276
08:13:03.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.641712 13776 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.126:37371
08:13:03.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.641789 13864 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.126:37371 every 8 connection(s)
08:13:03.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.644598 13776 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:13:03.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.649699 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 13776
08:13:03.650 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:03.650 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.13.124:46557
08:13:03.650 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.652256 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/logs
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:13:03.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.124:46557
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.124
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36079
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.124.keytab
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.124
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:13:03.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.658169 13865 sys_catalog.cc:263] Verifying existing consensus state
08:13:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.665328 13865 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Bootstrap starting.
08:13:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.716320 13865 log.cc:826] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Log is configured to *not* fsync() on all Append() calls
08:13:03.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.737098 13865 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:13:03.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.738137 13865 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Bootstrap complete.
08:13:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.770881 13865 raft_consensus.cc:357] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.774379 13865 raft_consensus.cc:738] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 268581e0456549218cd7fff0cb991913, State: Initialized, Role: FOLLOWER
08:13:03.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.775406 13865 consensus_queue.cc:260] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:03.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.778499 13869 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:03.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.779707 13869 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:13:03.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.780421 13865 sys_catalog.cc:564] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: configured and running, proceeding with master startup.
08:13:03.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.834841 13880 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:13:03.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.835386 13880 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: loading cluster ID for follower catalog manager: success
08:13:03.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.843971 13880 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: acquiring CA information for follower catalog manager: success
08:13:03.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:03 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839183, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.126@KRBTEST.COM for kudu/127.13.13.126@KRBTEST.COM
08:13:03.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.852577 13880 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:13:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.906188 13776 master_runner.cc:186] Error getting master registration for 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111),
08:13:03.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:03.934105 13776 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:13:04.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.093132 13867 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:13:04.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.093773 13867 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:13:04.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.094456 13867 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:13:04.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.127207 13867 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:13:04.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.127480 13867 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:13:04.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.127871 13867 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:13:04.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.128161 13867 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:13:04.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.128430 13867 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:13:04.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.128688 13867 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:13:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.128887 13867 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:13:04.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:04.171290 13867 master_runner.cc:386] Master server non-default flags:
08:13:04.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:04.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:04.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:04.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:04.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.124.keytab
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.124
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.124:46557
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:04.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.124
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36079
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/logs
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:13:04.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:04.173002 13867 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:13:04.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:04.174402 13867 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:13:04.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.192806 13891 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:04.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:04.955312 13894 raft_consensus.cc:491] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:13:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:04.955909 13894 raft_consensus.cc:513] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:04.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:04.987097 13894 leader_election.cc:290] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371), 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557)
08:13:05.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:05.009080 13723 leader_election.cc:336] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557): Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:13:05.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.012166 13840 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "268581e0456549218cd7fff0cb991913" is_pre_election: true
08:13:05.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.013276 13840 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dd4c83ae72e94b369d028ea29fc7e62c in term 1.
08:13:05.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.015483 13725 leader_election.cc:304] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 268581e0456549218cd7fff0cb991913, dd4c83ae72e94b369d028ea29fc7e62c; no voters: 3f97e33e85db48b3a34d544fa3495dd9
08:13:05.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.016458 13894 raft_consensus.cc:2802] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Leader pre-election won for term 2
08:13:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.016863 13894 raft_consensus.cc:491] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:13:05.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.017228 13894 raft_consensus.cc:3058] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 1 FOLLOWER]: Advancing to term 2
08:13:05.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.031230 13894 raft_consensus.cc:513] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.034463 13840 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "268581e0456549218cd7fff0cb991913"
08:13:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.035162 13840 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 1 FOLLOWER]: Advancing to term 2
08:13:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:05.051404 13723 leader_election.cc:336] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557): Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:13:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.052260 13894 leader_election.cc:290] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 2 election: Requested vote from peers 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371), 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557)
08:13:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.054981 13840 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dd4c83ae72e94b369d028ea29fc7e62c in term 2.
08:13:05.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.056406 13725 leader_election.cc:304] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 268581e0456549218cd7fff0cb991913, dd4c83ae72e94b369d028ea29fc7e62c; no voters: 3f97e33e85db48b3a34d544fa3495dd9
08:13:05.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.057773 13894 raft_consensus.cc:2802] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Leader election won for term 2
08:13:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.069929 13894 raft_consensus.cc:695] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 LEADER]: Becoming Leader. State: Replica: dd4c83ae72e94b369d028ea29fc7e62c, State: Running, Role: LEADER
08:13:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.071498 13894 consensus_queue.cc:237] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:05.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.098325 13899 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: New leader dd4c83ae72e94b369d028ea29fc7e62c. Latest consensus state: current_term: 2 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:05.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.099304 13899 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: LEADER
08:13:05.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.110661 13901 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:13:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.119547 13901 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:13:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.124182 13901 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:13:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.124691 13901 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:13:05.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.129737 13901 catalog_manager.cc:1506] Loading token signing keys...
08:13:05.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.134225 13901 catalog_manager.cc:5966] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Loaded TSK: 0
08:13:05.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.135780 13901 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:13:05.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.492547 13840 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 2 FOLLOWER]: Refusing update from remote peer dd4c83ae72e94b369d028ea29fc7e62c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
08:13:05.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.495090 13899 consensus_queue.cc:1035] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [LEADER]: Connected to new peer: Peer: permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
08:13:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.536125 13903 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: New leader dd4c83ae72e94b369d028ea29fc7e62c. Latest consensus state: current_term: 2 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:05.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.538832 13903 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:13:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.554344 13903 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.555055 13903 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:13:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.564868 13899 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:05.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.565642 13899 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: LEADER
08:13:05.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:05.600703 13723 consensus_peers.cc:487] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c -> Peer 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557): Couldn't send request to peer 3f97e33e85db48b3a34d544fa3495dd9. Status: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:13:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:05.589591 13889 debug-util.cc:398] Leaking SignalData structure 0x7b0800018780 after lost signal to thread 13867
08:13:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:05.671070 13867 thread.cc:640] GCE (cloud detector) Time spent creating pthread: real 1.480s user 0.539s sys 0.912s
08:13:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:05.671384 13867 thread.cc:606] GCE (cloud detector) Time spent starting thread: real 1.481s user 0.539s sys 0.912s
08:13:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:04.193308 13890 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:05.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:05.679975 13910 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:05.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.681826 13867 server_base.cc:1048] running on GCE node
08:13:05.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.683419 13867 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:13:05.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.686383 13867 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:13:05.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.687809 13867 hybrid_clock.cc:648] HybridClock initialized: now 1750839185687730 us; error 73 us; skew 500 ppm
08:13:05.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.688822 13867 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:13:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:05 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839185, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:13:05.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.695400 13867 init.cc:377] Logged in from keytab as kudu/127.13.13.124@KRBTEST.COM (short username kudu)
08:13:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.700727 13867 webserver.cc:469] Webserver started at http://127.13.13.124:36079/ using document root <none> and password file <none>
08:13:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.701719 13867 fs_manager.cc:362] Metadata directory not provided
08:13:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.701929 13867 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:13:05.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.709960 13867 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.002s
08:13:05.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.714654 13917 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:13:05.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.715821 13867 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.716177 13867 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f97e33e85db48b3a34d544fa3495dd9"
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:55 on dist-test-slave-nz51"
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.718103 13867 fs_report.cc:389] FS layout report
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/data
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:13:05.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:13:05.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:13:05.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:13:05.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:13:05.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:13:05.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.775079 13867 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:13:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.781131 13867 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:13:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.781589 13867 kserver.cc:163] Server-wide thread pool size limit: 3276
08:13:05.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.865795 13867 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.124:46557
08:13:05.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.865865 13968 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.124:46557 every 8 connection(s)
08:13:05.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.869839 13867 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:13:05.872 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:05.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.871691 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 13867
08:13:05.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.884650 13969 sys_catalog.cc:263] Verifying existing consensus state
08:13:05.886 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.13.125:37877(127.13.13.125:37877)] connecting to peer
08:13:05.888 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.13.126:37371(127.13.13.126:37371)] connecting to peer
08:13:05.891 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.13.124:46557(127.13.13.124:46557)] connecting to peer
08:13:05.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.895341 13969 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Bootstrap starting.
08:13:05.912 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.13.13.125:37877(127.13.13.125:37877)] Successfully connected to peer
08:13:05.914 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.13.13.124:46557(127.13.13.124:46557)] Successfully connected to peer
08:13:05.916 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
08:13:05.925 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.13.13.125
08:13:05.932 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.13.13.126:37371(127.13.13.126:37371)] Successfully connected to peer
08:13:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:05 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839176, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.13.125@KRBTEST.COM
08:13:05.955 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.13.13.126
08:13:05.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.959910 13969 log.cc:826] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Log is configured to *not* fsync() on all Append() calls
08:13:05.971 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.13.13.124
08:13:05.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:05 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839176, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.13.126@KRBTEST.COM
08:13:05.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.997920 13969 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:13:05.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:05.999014 13969 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Bootstrap complete.
08:13:06.003 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xa4273f3c, L:/127.0.0.1:50286 - R:/127.13.13.126:37371] using SASL/GSSAPI
08:13:06.017 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.13.124:46557(127.13.13.124:46557)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:13:06.017 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.13.124:46557: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
08:13:06.018 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.13.13.124:46557(127.13.13.124:46557)] handling channelInactive
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:06.020747 13976 negotiation.cc:337] Failed RPC negotiation. Trace:
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:05.901216 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:47222 (local address 127.13.13.124:46557)
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:05.920783 (+ 19567us) server_negotiation.cc:206] Beginning negotiation
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:05.920853 (+ 70us) server_negotiation.cc:399] Waiting for connection header
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:05.920981 (+ 128us) server_negotiation.cc:407] Connection header received
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:05.930123 (+ 9142us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:05.930347 (+ 224us) server_negotiation.cc:461] Received NEGOTIATE request from client
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:05.931200 (+ 853us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:05.931973 (+ 773us) server_negotiation.cc:226] Negotiated authn=SASL
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:05.980732 (+ 48759us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:05.983382 (+ 2650us) server_negotiation.cc:377] Sending TLS_HANDSHAKE NegotiatePB response
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:06.016508 (+ 33126us) server_negotiation.cc:365] Received TLS_HANDSHAKE NegotiatePB request
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:06.017235 (+ 727us) server_negotiation.cc:662] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:06.018733 (+ 1498us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:47222: BlockingRecv error: failed to read from TLS socket (remote: unknown): Cannot send after transport endpoint shutdown (error 108)
08:13:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":18378,"thread_start_us":25012,"threads_started":1}
08:13:06.024 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xdf877958, L:/127.0.0.1:34884 - R:/127.13.13.125:37877] using SASL/GSSAPI
08:13:06.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:13:06.032 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=160), Traces: [0ms] refreshing cache from master, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.13.125:37877, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.13.126:37371, [7ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.13.124:46557, [131ms] Sub RPC ConnectToMaster: received response from server master-127.13.13.126:37371: OK, [133ms] Sub RPC ConnectToMaster: received response from server master-127.13.13.124:46557: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [146ms] Sub RPC ConnectToMaster: received response from server master-127.13.13.125:37877: OK, deferred=null) after lookup
08:13:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.041214 13969 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:06.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.044587 13969 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3f97e33e85db48b3a34d544fa3495dd9, State: Initialized, Role: FOLLOWER
08:13:06.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.045747 13969 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:06.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.048709 13979 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:06.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.049579 13979 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: This master's current role is: FOLLOWER
08:13:06.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.050674 13969 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: configured and running, proceeding with master startup.
08:13:06.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.085435 13990 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:13:06.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.085928 13990 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: loading cluster ID for follower catalog manager: success
08:13:06.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.094534 13990 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: acquiring CA information for follower catalog manager: success
08:13:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.100493 13990 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:13:06.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.112892 13944 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 1 FOLLOWER]: Advancing to term 2
08:13:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.155623 13996 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.156725 13996 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: This master's current role is: FOLLOWER
08:13:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.160108 13979 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader dd4c83ae72e94b369d028ea29fc7e62c. Latest consensus state: current_term: 2 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.160761 13979 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: This master's current role is: FOLLOWER
08:13:06.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.503491 13894 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:06.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:06.504303 13894 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: LEADER
08:13:09.088 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.13.125:37877
08:13:09.088 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:13:09.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:09.088999 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 13705
08:13:09.112 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.13.13.125:37877(127.13.13.125:37877)] handling channelInactive
08:13:09.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.13.125:37877(127.13.13.125:37877)] cleaning up while in state READY due to: connection closed
08:13:09.112 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:09.113 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.13.126:37371
08:13:09.113 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:13:09.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:09.113657 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 13776
08:13:09.137 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.13.13.126:37371(127.13.13.126:37371)] handling channelInactive
08:13:09.137 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.13.126:37371(127.13.13.126:37371)] cleaning up while in state READY due to: connection closed
08:13:09.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:09.138 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.13.124:46557
08:13:09.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:13:09.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:09.138697 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 13867
08:13:09.161 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:09.161 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.13.125:37877
08:13:09.161 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:13:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:09.163132 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data
08:13:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:13:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:13:09.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/logs
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.125:37877
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.125
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32937
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.125.keytab
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.125
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:09.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:13:09.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.467370 14005 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:13:09.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.468003 14005 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:13:09.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.468891 14005 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:13:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.505394 14005 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:13:09.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.505700 14005 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:13:09.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.506100 14005 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:13:09.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.506314 14005 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:13:09.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.506500 14005 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:13:09.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.506677 14005 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:13:09.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.506810 14005 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:13:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:09.545467 14005 master_runner.cc:386] Master server non-default flags:
08:13:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.125.keytab
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.125
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.125:37877
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.125
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32937
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/logs
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:13:09.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:13:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:13:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:13:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:13:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:13:09.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:09.547140 14005 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:13:09.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:09.548216 14005 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:13:09.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.563889 14012 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.563903 14013 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:09.566675 14005 server_base.cc:1048] running on GCE node
08:13:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:09.564792 14015 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:10.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.698837 14005 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:13:10.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.701534 14005 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:13:10.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.702879 14005 hybrid_clock.cc:648] HybridClock initialized: now 1750839190702831 us; error 66 us; skew 500 ppm
08:13:10.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.703675 14005 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:13:10.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:10 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839190, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:13:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.709617 14005 init.cc:377] Logged in from keytab as kudu/127.13.13.125@KRBTEST.COM (short username kudu)
08:13:10.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.714730 14005 webserver.cc:469] Webserver started at http://127.13.13.125:32937/ using document root <none> and password file <none>
08:13:10.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.715719 14005 fs_manager.cc:362] Metadata directory not provided
08:13:10.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.715917 14005 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:13:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.724064 14005 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
08:13:10.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.729030 14023 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:13:10.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.730315 14005 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:13:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.730641 14005 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd4c83ae72e94b369d028ea29fc7e62c"
08:13:10.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:54 on dist-test-slave-nz51"
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.732546 14005 fs_report.cc:389] FS layout report
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/data
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:13:10.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:13:10.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.791078 14005 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:13:10.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.797217 14005 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:13:10.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.797731 14005 kserver.cc:163] Server-wide thread pool size limit: 3276
08:13:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.871393 14005 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.125:37877
08:13:10.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.871467 14074 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.125:37877 every 8 connection(s)
08:13:10.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.874389 14005 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:13:10.877 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:10.877 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.13.126:37371
08:13:10.877 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:13:10.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.876513 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 14005
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.878892 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/logs
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:10.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.126:37371
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.126
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42523
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.126.keytab
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.126
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:13:10.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.886389 14075 sys_catalog.cc:263] Verifying existing consensus state
08:13:10.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.892944 14075 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Bootstrap starting.
08:13:10.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.931301 14075 log.cc:826] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Log is configured to *not* fsync() on all Append() calls
08:13:10.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.947580 14075 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:13:10.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.948432 14075 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Bootstrap complete.
08:13:10.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.971724 14075 raft_consensus.cc:357] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:10.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.974027 14075 raft_consensus.cc:738] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd4c83ae72e94b369d028ea29fc7e62c, State: Initialized, Role: FOLLOWER
08:13:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.974785 14075 consensus_queue.cc:260] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:10.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.977241 14079 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:10.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.978077 14079 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: FOLLOWER
08:13:10.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:10.978948 14075 sys_catalog.cc:564] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: configured and running, proceeding with master startup.
08:13:11.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:11.015214 14090 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:13:11.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:11.015498 14090 catalog_manager.cc:1554] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: loading cluster ID for follower catalog manager: success
08:13:11.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:11.024286 14090 catalog_manager.cc:1576] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: acquiring CA information for follower catalog manager: success
08:13:11.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:11.030476 14090 catalog_manager.cc:1604] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:13:11.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:11.004881 14005 master_runner.cc:186] Error getting master registration for 127.13.13.126:37371: Network error: Client connection negotiation failed: client connection to 127.13.13.126:37371: connect: Connection refused (error 111),
08:13:11.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:11.034540 14005 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:13:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.205710 14077 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:13:11.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.206360 14077 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:13:11.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.207314 14077 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:13:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.240792 14077 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:13:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.241073 14077 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:13:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.241495 14077 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:13:11.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.241796 14077 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:13:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.242082 14077 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:13:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.242352 14077 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:13:11.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.242552 14077 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:13:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:11.280300 14077 master_runner.cc:386] Master server non-default flags:
08:13:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data
08:13:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.126.keytab
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.126
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.126:37371
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.126
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42523
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/logs
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:13:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:13:11.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:11.281988 14077 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:13:11.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:11.283069 14077 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:13:11.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.299461 14099 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:12.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.412178 14105 raft_consensus.cc:491] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:13:12.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.412875 14105 raft_consensus.cc:513] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:12.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.417181 14105 leader_election.cc:290] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371), 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557)
08:13:12.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:12.432217 14024 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:13:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:12.437861 14024 leader_election.cc:336] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557): Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:13:12.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:12.438498 14026 leader_election.cc:336] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371): Network error: Client connection negotiation failed: client connection to 127.13.13.126:37371: connect: Connection refused (error 111)
08:13:12.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.438933 14026 leader_election.cc:304] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dd4c83ae72e94b369d028ea29fc7e62c; no voters: 268581e0456549218cd7fff0cb991913, 3f97e33e85db48b3a34d544fa3495dd9
08:13:12.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.440176 14105 raft_consensus.cc:2747] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
08:13:12.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.299788 14098 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:12.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:11.302241 14101 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:12.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:11.302778 14077 server_base.cc:1048] running on GCE node
08:13:12.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.453970 14077 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:13:12.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.457012 14077 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:13:12.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.458441 14077 hybrid_clock.cc:648] HybridClock initialized: now 1750839192458458 us; error 121 us; skew 500 ppm
08:13:12.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.459313 14077 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:13:12.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:12 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839192, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:13:12.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.470451 14077 init.cc:377] Logged in from keytab as kudu/127.13.13.126@KRBTEST.COM (short username kudu)
08:13:12.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.477200 14077 webserver.cc:469] Webserver started at http://127.13.13.126:42523/ using document root <none> and password file <none>
08:13:12.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.478277 14077 fs_manager.cc:362] Metadata directory not provided
08:13:12.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.478626 14077 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:13:12.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.490111 14077 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.005s sys 0.000s
08:13:12.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.495939 14114 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:13:12.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.497318 14077 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
08:13:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.497735 14077 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "268581e0456549218cd7fff0cb991913"
08:13:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:52 on dist-test-slave-nz51"
08:13:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.500026 14077 fs_report.cc:389] FS layout report
08:13:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:13:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/data
08:13:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:13:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:13:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:13:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:13:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:13:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:13:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:13:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:13:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:13:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:13:12.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:13:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.566960 14077 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:13:12.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.572703 14077 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:13:12.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.573112 14077 kserver.cc:163] Server-wide thread pool size limit: 3276
08:13:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.659840 14077 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.126:37371
08:13:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.659926 14165 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.126:37371 every 8 connection(s)
08:13:12.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.663031 14077 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:13:12.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.666563 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 14077
08:13:12.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:12.667 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.13.124:46557
08:13:12.667 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.669078 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/logs
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:13:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.124:46557
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.124
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36079
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.124.keytab
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.124
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:12.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:13:12.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.682354 14166 sys_catalog.cc:263] Verifying existing consensus state
08:13:12.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.688066 14166 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Bootstrap starting.
08:13:12.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.728492 14166 log.cc:826] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Log is configured to *not* fsync() on all Append() calls
08:13:12.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.746403 14166 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:13:12.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.747282 14166 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Bootstrap complete.
08:13:12.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.770391 14166 raft_consensus.cc:357] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:12.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.773666 14166 raft_consensus.cc:738] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 268581e0456549218cd7fff0cb991913, State: Initialized, Role: FOLLOWER
08:13:12.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.774667 14166 consensus_queue.cc:260] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:12.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.783071 14170 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:12.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.784161 14170 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:13:12.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.785198 14166 sys_catalog.cc:564] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: configured and running, proceeding with master startup.
08:13:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.820725 14181 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:13:12.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.821012 14181 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: loading cluster ID for follower catalog manager: success
08:13:12.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.836144 14181 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: acquiring CA information for follower catalog manager: success
08:13:12.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.846957 14181 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:13:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.916414 14077 master_runner.cc:186] Error getting master registration for 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111),
08:13:12.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:12.942253 14077 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:13:13.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.181947 14168 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:13:13.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.182719 14168 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:13:13.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.183754 14168 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:13:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.239930 14168 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:13:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.240278 14168 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:13:13.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.240751 14168 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:13:13.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.241099 14168 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:13:13.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.241457 14168 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:13:13.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.241741 14168 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:13:13.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.241956 14168 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:13:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.278767 14168 master_runner.cc:386] Master server non-default flags:
08:13:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data
08:13:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:13:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:13.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.124.keytab
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.124
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.124:46557
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.124
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36079
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/logs
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:13:13.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.280398 14168 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:13:13.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.281587 14168 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:13:13.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.298537 14194 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:13.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.958353 14197 raft_consensus.cc:491] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:13:13.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.959350 14197 raft_consensus.cc:513] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:13.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.973690 14197 leader_election.cc:290] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371), 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557)
08:13:13.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.986240 14141 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "268581e0456549218cd7fff0cb991913" is_pre_election: true
08:13:13.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.987273 14141 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dd4c83ae72e94b369d028ea29fc7e62c in term 2.
08:13:13.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.988710 14026 leader_election.cc:304] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 268581e0456549218cd7fff0cb991913, dd4c83ae72e94b369d028ea29fc7e62c; no voters:
08:13:13.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.990016 14197 raft_consensus.cc:2802] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Leader pre-election won for term 3
08:13:13.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.990633 14197 raft_consensus.cc:491] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:13:13.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:13.990978 14197 raft_consensus.cc:3058] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 2 FOLLOWER]: Advancing to term 3
08:13:14.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.002128 14197 raft_consensus.cc:513] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:14.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:14.006517 14024 leader_election.cc:336] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557): Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:13:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.009582 14141 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "268581e0456549218cd7fff0cb991913"
08:13:14.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.010212 14141 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 2 FOLLOWER]: Advancing to term 3
08:13:14.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.022071 14141 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dd4c83ae72e94b369d028ea29fc7e62c in term 3.
08:13:14.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.024307 14197 leader_election.cc:290] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 3 election: Requested vote from peers 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371), 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557)
08:13:14.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.025942 14026 leader_election.cc:304] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 268581e0456549218cd7fff0cb991913, dd4c83ae72e94b369d028ea29fc7e62c; no voters:
08:13:14.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.027001 14197 raft_consensus.cc:2802] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 3 FOLLOWER]: Leader election won for term 3
08:13:14.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:14.029996 14024 leader_election.cc:336] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557): Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:13:14.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.037961 14197 raft_consensus.cc:695] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 3 LEADER]: Becoming Leader. State: Replica: dd4c83ae72e94b369d028ea29fc7e62c, State: Running, Role: LEADER
08:13:14.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.039675 14197 consensus_queue.cc:237] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.057268 14202 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: New leader dd4c83ae72e94b369d028ea29fc7e62c. Latest consensus state: current_term: 3 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:14.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.058159 14202 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: LEADER
08:13:14.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.068738 14204 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:13:14.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.078192 14204 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:13:14.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.082898 14204 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:13:14.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.084036 14204 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:13:14.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.089200 14204 catalog_manager.cc:1506] Loading token signing keys...
08:13:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.093693 14204 catalog_manager.cc:5966] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Loaded TSK: 0
08:13:14.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.096794 14204 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:13:14.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.508992 14141 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 3 FOLLOWER]: Refusing update from remote peer dd4c83ae72e94b369d028ea29fc7e62c: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
08:13:14.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.514299 14197 consensus_queue.cc:1035] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [LEADER]: Connected to new peer: Peer: permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
08:13:14.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:14.522478 14024 consensus_peers.cc:487] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c -> Peer 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557): Couldn't send request to peer 3f97e33e85db48b3a34d544fa3495dd9. Status: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:13:14.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.559818 14197 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:14.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.560583 14197 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: LEADER
08:13:14.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.569043 14205 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: New leader dd4c83ae72e94b369d028ea29fc7e62c. Latest consensus state: current_term: 3 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:14.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.569212 14209 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:14.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.570115 14209 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:13:14.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.570127 14205 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:13:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.300536 14191 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:14.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:13.298584 14192 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:14.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:14.739863 14193 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1438 milliseconds
08:13:14.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.740025 14168 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:13:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.741555 14168 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:13:14.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.744774 14168 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:13:14.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.746291 14168 hybrid_clock.cc:648] HybridClock initialized: now 1750839194746149 us; error 160 us; skew 500 ppm
08:13:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.747367 14168 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:13:14.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:14 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839194, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:13:14.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.754534 14168 init.cc:377] Logged in from keytab as kudu/127.13.13.124@KRBTEST.COM (short username kudu)
08:13:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.760174 14168 webserver.cc:469] Webserver started at http://127.13.13.124:36079/ using document root <none> and password file <none>
08:13:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.761525 14168 fs_manager.cc:362] Metadata directory not provided
08:13:14.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.761826 14168 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:13:14.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.772778 14168 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
08:13:14.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.778614 14215 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:13:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.779949 14168 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
08:13:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.780391 14168 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f97e33e85db48b3a34d544fa3495dd9"
08:13:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:55 on dist-test-slave-nz51"
08:13:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.783159 14168 fs_report.cc:389] FS layout report
08:13:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:13:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/data
08:13:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:13:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:13:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:13:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:13:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:13:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:13:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:13:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:13:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:13:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:13:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:13:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.857980 14168 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:13:14.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.863842 14168 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:13:14.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.864328 14168 kserver.cc:163] Server-wide thread pool size limit: 3276
08:13:14.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.948865 14168 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.124:46557
08:13:14.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.948997 14266 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.124:46557 every 8 connection(s)
08:13:14.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.952342 14168 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:13:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.960675 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 14168
08:13:14.961 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:14.962 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.13.125:37877(127.13.13.125:37877)] connecting to peer
08:13:14.964 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.13.13.125:37877(127.13.13.125:37877)] Successfully connected to peer
08:13:14.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.967108 14267 sys_catalog.cc:263] Verifying existing consensus state
08:13:14.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:14.977125 14267 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Bootstrap starting.
08:13:14.989 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x182aca83, L:/127.0.0.1:56232 - R:/127.13.13.125:37877] using TOKEN/null
08:13:14.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:14 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839190, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.125@KRBTEST.COM for kudu/127.13.13.124@KRBTEST.COM
08:13:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.037228 14267 log.cc:826] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Log is configured to *not* fsync() on all Append() calls
08:13:15.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.059875 14267 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:13:15.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.060974 14267 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Bootstrap complete.
08:13:15.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.090197 14267 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:15.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.093787 14267 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3f97e33e85db48b3a34d544fa3495dd9, State: Initialized, Role: FOLLOWER
08:13:15.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.094817 14267 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:15.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.098006 14272 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.098837 14272 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: This master's current role is: FOLLOWER
08:13:15.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.100590 14267 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: configured and running, proceeding with master startup.
08:13:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.101181 14242 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 2 FOLLOWER]: Advancing to term 3
08:13:15.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.125129 14281 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:15.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.126622 14281 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: This master's current role is: FOLLOWER
08:13:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.130412 14272 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader dd4c83ae72e94b369d028ea29fc7e62c. Latest consensus state: current_term: 3 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:15.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.131021 14272 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: This master's current role is: FOLLOWER
08:13:15.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.159266 14286 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:13:15.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.159806 14286 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: loading cluster ID for follower catalog manager: success
08:13:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:15 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839194, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.124@KRBTEST.COM for kudu/127.13.13.126@KRBTEST.COM
08:13:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.168172 14286 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: acquiring CA information for follower catalog manager: success
08:13:15.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.174747 14286 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:13:15.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.416059 14202 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:15.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:15.416783 14202 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: LEADER
08:13:18.042 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.13.125:37877
08:13:18.042 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:13:18.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:18.042687 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 14005
08:13:18.065 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.13.13.125:37877(127.13.13.125:37877)] handling channelInactive
08:13:18.066 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.13.125:37877(127.13.13.125:37877)] cleaning up while in state READY due to: connection closed
08:13:18.066 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:18.066 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.13.126:37371
08:13:18.066 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:13:18.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:18.067138 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 14077
08:13:18.089 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:18.089 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.13.124:46557
08:13:18.090 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:13:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:18.090365 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 14168
08:13:18.111 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:18.112 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.13.125:37877
08:13:18.120 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:18.121899 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/logs
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:13:18.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.125:37877
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.125
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32937
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.125.keytab
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.125
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:18.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:13:18.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.153227 12720 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac5e0 after lost signal to thread 12657
08:13:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.153972 12720 debug-util.cc:398] Leaking SignalData structure 0x7b0800079fe0 after lost signal to thread 12723
08:13:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.449954 14302 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:13:18.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.450477 14302 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:13:18.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.451076 14302 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:13:18.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.483410 14302 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:13:18.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.483647 14302 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:13:18.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.483920 14302 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:13:18.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.484133 14302 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:13:18.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.484329 14302 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:13:18.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.484514 14302 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:13:18.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.484655 14302 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:18.525008 14302 master_runner.cc:386] Master server non-default flags:
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.125.keytab
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.125
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.125:37877
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:13:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:13:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:13:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.125
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=32937
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/logs
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:18.526453 14302 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:13:18.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:18.527681 14302 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:13:18.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.546145 14309 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:19.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.547132 14308 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:19.675474 14310 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:13:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.675563 14302 server_base.cc:1043] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:13:19.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:18.546301 14311 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:19.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.679888 14302 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:13:19.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.682888 14302 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:13:19.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.684412 14302 hybrid_clock.cc:648] HybridClock initialized: now 1750839199684380 us; error 46 us; skew 500 ppm
08:13:19.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.685214 14302 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:13:19.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:19 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839199, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:13:19.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.694980 14302 init.cc:377] Logged in from keytab as kudu/127.13.13.125@KRBTEST.COM (short username kudu)
08:13:19.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.700439 14302 webserver.cc:469] Webserver started at http://127.13.13.125:32937/ using document root <none> and password file <none>
08:13:19.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.701341 14302 fs_manager.cc:362] Metadata directory not provided
08:13:19.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.701576 14302 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:13:19.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.709688 14302 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
08:13:19.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.714360 14321 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:13:19.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.715428 14302 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
08:13:19.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.715718 14302 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:19.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd4c83ae72e94b369d028ea29fc7e62c"
08:13:19.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:54 on dist-test-slave-nz51"
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.717664 14302 fs_report.cc:389] FS layout report
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/wal
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/data
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:13:19.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:13:19.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:13:19.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:13:19.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.778823 14302 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:13:19.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.784371 14302 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:13:19.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.784823 14302 kserver.cc:163] Server-wide thread pool size limit: 3276
08:13:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.864166 14302 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.125:37877
08:13:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.864248 14372 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.125:37877 every 8 connection(s)
08:13:19.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.867156 14302 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-1/data/info.pb
08:13:19.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.875085 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 14302
08:13:19.876 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:19.876 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.13.126:37371
08:13:19.876 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.878038 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/logs
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:13:19.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.126:37371
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.126
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42523
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.126.keytab
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.126
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:13:19.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.878638 14373 sys_catalog.cc:263] Verifying existing consensus state
08:13:19.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.885043 14373 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Bootstrap starting.
08:13:19.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.922025 14373 log.cc:826] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Log is configured to *not* fsync() on all Append() calls
08:13:19.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.939990 14373 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:13:19.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.940701 14373 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Bootstrap complete.
08:13:19.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.963042 14373 raft_consensus.cc:357] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:19.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.965713 14373 raft_consensus.cc:738] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: dd4c83ae72e94b369d028ea29fc7e62c, State: Initialized, Role: FOLLOWER
08:13:19.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.966648 14373 consensus_queue.cc:260] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.969125 14377 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.970016 14377 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: FOLLOWER
08:13:19.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.971081 14373 sys_catalog.cc:564] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: configured and running, proceeding with master startup.
08:13:20.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:19.999452 14388 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:13:20.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:20.004597 14388 catalog_manager.cc:1554] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: loading cluster ID for follower catalog manager: success
08:13:20.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:20.012328 14388 catalog_manager.cc:1576] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: acquiring CA information for follower catalog manager: success
08:13:20.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:20.018059 14388 catalog_manager.cc:1604] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:13:20.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:20.009332 14302 master_runner.cc:186] Error getting master registration for 127.13.13.126:37371: Network error: Client connection negotiation failed: client connection to 127.13.13.126:37371: connect: Connection refused (error 111),
08:13:20.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:20.029834 14302 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:13:20.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.205617 14375 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:13:20.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.206236 14375 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:13:20.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.206945 14375 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:13:20.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.239866 14375 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:13:20.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.240147 14375 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:13:20.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.240576 14375 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:13:20.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.240875 14375 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:13:20.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.241156 14375 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:13:20.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.241443 14375 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:13:20.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.241673 14375 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:13:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:20.279846 14375 master_runner.cc:386] Master server non-default flags:
08:13:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data
08:13:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:13:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:20.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.126.keytab
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.126
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.126:37371
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.126
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42523
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/logs
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:13:20.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:13:20.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:13:20.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:13:20.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:13:20.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:20.281514 14375 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:13:20.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:20.282734 14375 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:13:20.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.298635 14397 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:20.300386 14375 server_base.cc:1048] running on GCE node
08:13:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.301496 14399 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:21.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:20.300216 14396 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:21.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.496116 14375 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:13:21.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.499277 14375 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:13:21.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.500694 14375 hybrid_clock.cc:648] HybridClock initialized: now 1750839201500640 us; error 73 us; skew 500 ppm
08:13:21.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.501519 14375 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:13:21.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:21 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839201, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:13:21.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.507499 14375 init.cc:377] Logged in from keytab as kudu/127.13.13.126@KRBTEST.COM (short username kudu)
08:13:21.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.512955 14375 webserver.cc:469] Webserver started at http://127.13.13.126:42523/ using document root <none> and password file <none>
08:13:21.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.513917 14375 fs_manager.cc:362] Metadata directory not provided
08:13:21.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.514137 14375 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:13:21.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.522902 14375 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.001s
08:13:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.528041 14407 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:13:21.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.529142 14375 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
08:13:21.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.529528 14375 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:21.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "268581e0456549218cd7fff0cb991913"
08:13:21.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:52 on dist-test-slave-nz51"
08:13:21.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.532116 14375 fs_report.cc:389] FS layout report
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/wal
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/data
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:13:21.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:13:21.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:13:21.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:13:21.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:13:21.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.594671 14375 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:13:21.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.600430 14375 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:13:21.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.600896 14375 kserver.cc:163] Server-wide thread pool size limit: 3276
08:13:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.685789 14375 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.126:37371
08:13:21.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.685874 14458 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.126:37371 every 8 connection(s)
08:13:21.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.688671 14375 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-0/data/info.pb
08:13:21.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.692369 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 14375
08:13:21.693 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:21.693 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.13.124:46557
08:13:21.693 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:13:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.695327 13365 external_mini_cluster.cc:1351] Running /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu
08:13:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data
08:13:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:13:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:13:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/logs
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.124:46557
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.124
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36079
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.124.keytab
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.124
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:21.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:13:21.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.702940 14459 sys_catalog.cc:263] Verifying existing consensus state
08:13:21.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.708732 14459 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Bootstrap starting.
08:13:21.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.754917 14459 log.cc:826] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Log is configured to *not* fsync() on all Append() calls
08:13:21.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.786094 14459 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:13:21.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.787098 14459 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: Bootstrap complete.
08:13:21.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.818452 14459 raft_consensus.cc:357] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:21.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.821674 14459 raft_consensus.cc:738] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 268581e0456549218cd7fff0cb991913, State: Initialized, Role: FOLLOWER
08:13:21.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.822782 14459 consensus_queue.cc:260] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:21.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.840528 14459 sys_catalog.cc:564] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: configured and running, proceeding with master startup.
08:13:21.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.838469 14463 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:21.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.842830 14463 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:13:21.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.843958 14464 raft_consensus.cc:491] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:13:21.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.844529 14464 raft_consensus.cc:513] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:21.870754 14322 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:13:21.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:21 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839199, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.125@KRBTEST.COM for kudu/127.13.13.126@KRBTEST.COM
08:13:21.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.885339 14464 leader_election.cc:290] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371), 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557)
08:13:21.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:21.903620 14322 leader_election.cc:336] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557): Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:13:21.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.912632 14434 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "268581e0456549218cd7fff0cb991913" is_pre_election: true
08:13:21.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.913630 14434 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dd4c83ae72e94b369d028ea29fc7e62c in term 3.
08:13:21.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.915747 14324 leader_election.cc:304] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 268581e0456549218cd7fff0cb991913, dd4c83ae72e94b369d028ea29fc7e62c; no voters: 3f97e33e85db48b3a34d544fa3495dd9
08:13:21.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.917012 14464 raft_consensus.cc:2802] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 3 FOLLOWER]: Leader pre-election won for term 4
08:13:21.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.917348 14464 raft_consensus.cc:491] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:13:21.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.917680 14464 raft_consensus.cc:3058] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 3 FOLLOWER]: Advancing to term 4
08:13:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.928237 14464 raft_consensus.cc:513] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:21.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.936331 14434 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "268581e0456549218cd7fff0cb991913"
08:13:21.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.936892 14434 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 3 FOLLOWER]: Advancing to term 4
08:13:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.947340 14434 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dd4c83ae72e94b369d028ea29fc7e62c in term 4.
08:13:21.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.948524 14324 leader_election.cc:304] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 268581e0456549218cd7fff0cb991913, dd4c83ae72e94b369d028ea29fc7e62c; no voters:
08:13:21.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.955367 14482 raft_consensus.cc:2802] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 4 FOLLOWER]: Leader election won for term 4
08:13:21.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.955963 14464 leader_election.cc:290] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 4 election: Requested vote from peers 268581e0456549218cd7fff0cb991913 (127.13.13.126:37371), 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557)
08:13:21.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:21.958563 14322 leader_election.cc:336] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557): Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111)
08:13:21.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:21 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839201, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.126@KRBTEST.COM for kudu/127.13.13.126@KRBTEST.COM
08:13:21.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.963065 14479 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:13:21.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.964879 14479 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: loading cluster ID for follower catalog manager: success
08:13:21.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.984395 14482 raft_consensus.cc:695] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [term 4 LEADER]: Becoming Leader. State: Replica: dd4c83ae72e94b369d028ea29fc7e62c, State: Running, Role: LEADER
08:13:21.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.986697 14482 consensus_queue.cc:237] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:21.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:21.988822 14479 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: acquiring CA information for follower catalog manager: success
08:13:22.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.006603 14479 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:13:22.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:22 dist-test-slave-nz51 krb5kdc[13376](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839201, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.126@KRBTEST.COM for kudu/127.13.13.125@KRBTEST.COM
08:13:22.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.016574 14464 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: New leader dd4c83ae72e94b369d028ea29fc7e62c. Latest consensus state: current_term: 4 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:22.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.017306 14464 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: LEADER
08:13:22.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.031520 14486 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
08:13:22.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.045477 14486 catalog_manager.cc:1486] Initializing Kudu cluster ID...
08:13:22.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.050328 14486 catalog_manager.cc:1261] Loaded cluster ID: 62834fcd38284bccaf7a0f6b3bdc8a25
08:13:22.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.050809 14486 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
08:13:22.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.062601 14486 catalog_manager.cc:1506] Loading token signing keys...
08:13:22.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.067984 14486 catalog_manager.cc:5966] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c: Loaded TSK: 0
08:13:22.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.069511 14486 catalog_manager.cc:1516] Initializing in-progress tserver states...
08:13:22.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.050796 14375 master_runner.cc:186] Error getting master registration for 127.13.13.124:46557: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111),
08:13:22.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.086687 14375 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
08:13:22.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.210614 14461 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
08:13:22.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.211221 14461 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:13:22.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.211930 14461 flags.cc:425] Enabled unsafe flag: --never_fsync=true
08:13:22.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.243801 14461 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
08:13:22.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.244081 14461 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
08:13:22.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.244464 14461 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
08:13:22.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.244731 14461 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
08:13:22.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.244993 14461 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
08:13:22.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.245267 14461 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
08:13:22.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.245494 14461 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44789/jwks.json
08:13:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.284368 14461 master_runner.cc:386] Master server non-default flags:
08:13:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:13:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.13.84:38681
08:13:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:13:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:13:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data
08:13:22.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.13.126:37371,127.13.13.125:37877,127.13.13.124:46557
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/krb5kdc/kudu_127.13.13.124.keytab
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.13.124
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.13.124:46557
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44789/jwks.json
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.13.124
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36079
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/logs
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 6d72d4a242076468501f3430b9a2cd050c634be2
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:13:22.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 25 Jun 2025 07:43:21 UTC on 5fd53c4cbb9d
08:13:22.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 6761
08:13:22.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:13:22.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.285982 14461 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:13:22.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.287253 14461 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:13:22.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.302723 14493 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:22.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.368531 14434 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [term 4 FOLLOWER]: Refusing update from remote peer dd4c83ae72e94b369d028ea29fc7e62c: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
08:13:22.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.370070 14464 consensus_queue.cc:1035] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [LEADER]: Connected to new peer: Peer: permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
08:13:22.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.381443 14499 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: New leader dd4c83ae72e94b369d028ea29fc7e62c. Latest consensus state: current_term: 4 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:22.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.381948 14499 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:13:22.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.385926 14483 sys_catalog.cc:455] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:22.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.386482 14483 sys_catalog.cc:458] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c [sys.catalog]: This master's current role is: LEADER
08:13:22.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.390913 14322 consensus_peers.cc:487] T 00000000000000000000000000000000 P dd4c83ae72e94b369d028ea29fc7e62c -> Peer 3f97e33e85db48b3a34d544fa3495dd9 (127.13.13.124:46557): Couldn't send request to peer 3f97e33e85db48b3a34d544fa3495dd9. Status: Network error: Client connection negotiation failed: client connection to 127.13.13.124:46557: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:13:22.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.393019 14499 sys_catalog.cc:455] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:22.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.393967 14499 sys_catalog.cc:458] T 00000000000000000000000000000000 P 268581e0456549218cd7fff0cb991913 [sys.catalog]: This master's current role is: FOLLOWER
08:13:23.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:22.306536 14461 server_base.cc:1048] running on GCE node
08:13:23.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.306048 14492 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:22.304301 14495 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:13:23.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.486176 14461 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:13:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.490463 14461 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:13:23.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.491896 14461 hybrid_clock.cc:648] HybridClock initialized: now 1750839203491807 us; error 105 us; skew 500 ppm
08:13:23.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.492764 14461 server_base.cc:848] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:13:23.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 25 08:13:23 dist-test-slave-nz51 krb5kdc[13376](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1750839203, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.13.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:13:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.498587 14461 init.cc:377] Logged in from keytab as kudu/127.13.13.124@KRBTEST.COM (short username kudu)
08:13:23.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.504240 14461 webserver.cc:469] Webserver started at http://127.13.13.124:36079/ using document root <none> and password file <none>
08:13:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.505331 14461 fs_manager.cc:362] Metadata directory not provided
08:13:23.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.505597 14461 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:13:23.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.514346 14461 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.003s
08:13:23.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.520068 14510 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:13:23.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.521317 14461 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.002s
08:13:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.521754 14461 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data,/tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3f97e33e85db48b3a34d544fa3495dd9"
08:13:23.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-06-25 08:12:55 on dist-test-slave-nz51"
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.524572 14461 fs_report.cc:389] FS layout report
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/wal
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/data
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:13:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:13:23.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.598111 14461 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:13:23.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.604107 14461 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:13:23.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.604583 14461 kserver.cc:163] Server-wide thread pool size limit: 3276
08:13:23.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.687083 14461 rpc_server.cc:307] RPC server started. Bound to: 127.13.13.124:46557
08:13:23.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.687165 14561 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.13.124:46557 every 8 connection(s)
08:13:23.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.690090 14461 server_base.cc:1180] Dumped server information to /tmp/dist-test-taskzFzfv3/test-tmp/mini-kudu-cluster6589963674367928150/master-2/data/info.pb
08:13:23.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.697383 13365 external_mini_cluster.cc:1413] Started /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu as pid 14461
08:13:23.698 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:13:23.698 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.13.125:37877(127.13.13.125:37877)] connecting to peer
08:13:23.702 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.13.13.125:37877(127.13.13.125:37877)] Successfully connected to peer
08:13:23.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.703626 14562 sys_catalog.cc:263] Verifying existing consensus state
08:13:23.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.715665 14562 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Bootstrap starting.
08:13:23.733 [DEBUG - kudu-nio-0] (Negotiator.java:352) peer /127.13.13.125:37877 sent connection negotiation error: Not authorized: token expired
08:13:23.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:23.733143 14564 negotiation.cc:344] Unauthorized connection attempt: Server connection negotiation failed: server connection from 127.0.0.1:43464: token expired
08:13:23.735 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.13.13.125:37877(127.13.13.125:37877)] handling channelInactive
08:13:23.735 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.13.125:37877(127.13.13.125:37877)] cleaning up while in state NEGOTIATION_FAILED due to: connection closed
08:13:23.736 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.13.13.125:37877(127.13.13.125:37877)] connecting to peer
08:13:23.738 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.13.13.125:37877(127.13.13.125:37877)] Successfully connected to peer
08:13:23.741 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.13.125:37877(127.13.13.125:37877)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:13:23.742 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.13.125:37877: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:13:23.742 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.13.13.125:37877(127.13.13.125:37877)] handling channelInactive
08:13:23.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:23.745353 14564 negotiation.cc:337] Failed RPC negotiation. Trace:
08:13:23.745 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.13.13.126:37371(127.13.13.126:37371)] connecting to peer
08:13:23.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.738182 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43476 (local address 127.13.13.125:37877)
08:13:23.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.740529 (+ 2347us) server_negotiation.cc:206] Beginning negotiation
08:13:23.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.740556 (+ 27us) server_negotiation.cc:399] Waiting for connection header
08:13:23.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.740600 (+ 44us) server_negotiation.cc:407] Connection header received
08:13:23.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.740731 (+ 131us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
08:13:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.740761 (+ 30us) server_negotiation.cc:461] Received NEGOTIATE request from client
08:13:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.740996 (+ 235us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
08:13:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.743565 (+ 2569us) server_negotiation.cc:226] Negotiated authn=SASL
08:13:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.744145 (+ 580us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43476: BlockingRecv error: recv got EOF from 127.0.0.1:43476 (error 108)
08:13:23.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1762}
08:13:23.746 [DEBUG - kudu-nio-0] (Connection.java:748) [peer master-127.13.13.124:46557(127.13.13.124:46557)] connecting to peer
08:13:23.747 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.13.13.126:37371(127.13.13.126:37371)] Successfully connected to peer
08:13:23.751 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.13.13.124:46557(127.13.13.124:46557)] Successfully connected to peer
08:13:23.776 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.13.126:37371(127.13.13.126:37371)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:13:23.777 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.13.126:37371: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:13:23.777 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.13.13.126:37371(127.13.13.126:37371)] handling channelInactive
08:13:23.780 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.13.124:46557(127.13.13.124:46557)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:13:23.780 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.13.124:46557: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials
08:13:23.781 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.13.13.124:46557(127.13.13.124:46557)] handling channelInactive
08:13:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:23.785172 14566 negotiation.cc:337] Failed RPC negotiation. Trace:
08:13:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.752060 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41226 (local address 127.13.13.124:46557)
08:13:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.763337 (+ 11277us) server_negotiation.cc:206] Beginning negotiation
08:13:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.763379 (+ 42us) server_negotiation.cc:399] Waiting for connection header
08:13:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.763465 (+ 86us) server_negotiation.cc:407] Connection header received
08:13:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.778690 (+ 15225us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
08:13:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.778752 (+ 62us) server_negotiation.cc:461] Received NEGOTIATE request from client
08:13:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.779287 (+ 535us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
08:13:23.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.779802 (+ 515us) server_negotiation.cc:226] Negotiated authn=SASL
08:13:23.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.782875 (+ 3073us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41226: BlockingRecv error: recv got EOF from 127.0.0.1:41226 (error 108)
08:13:23.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":10170,"thread_start_us":9546,"threads_started":1}
08:13:23.786 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.13.126:37371(127.13.13.126:37371)] disconnecting while in state TERMINATED
08:13:23.786 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.13.124:46557(127.13.13.124:46557)] disconnecting while in state TERMINATED
08:13:23.787 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.13.125:37877(127.13.13.125:37877)] disconnecting while in state TERMINATED
08:13:23.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250625 08:13:23.786581 14565 negotiation.cc:337] Failed RPC negotiation. Trace:
08:13:23.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.747677 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:42896 (local address 127.13.13.126:37371)
08:13:23.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.775468 (+ 27791us) server_negotiation.cc:206] Beginning negotiation
08:13:23.787 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:13:23.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.775493 (+ 25us) server_negotiation.cc:399] Waiting for connection header
08:13:23.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.775548 (+ 55us) server_negotiation.cc:407] Connection header received
08:13:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.775691 (+ 143us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
08:13:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.775716 (+ 25us) server_negotiation.cc:461] Received NEGOTIATE request from client
08:13:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.775930 (+ 214us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
08:13:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.778009 (+ 2079us) server_negotiation.cc:226] Negotiated authn=SASL
08:13:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0625 08:13:23.778628 (+ 619us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:42896: BlockingRecv error: recv got EOF from 127.0.0.1:42896 (error 108)
08:13:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":27189,"thread_start_us":26878,"threads_started":1}
08:13:23.791 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:13:23.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.795730 14562 log.cc:826] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Log is configured to *not* fsync() on all Append() calls
08:13:23.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.803610 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 14375
08:13:23.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.815824 14562 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:13:23.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.816776 14562 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9: Bootstrap complete.
08:13:23.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.827992 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 14302
08:13:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.845104 14562 raft_consensus.cc:357] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:23.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.848441 14562 raft_consensus.cc:738] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3f97e33e85db48b3a34d544fa3495dd9, State: Initialized, Role: FOLLOWER
08:13:23.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.849412 14562 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } }
08:13:23.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.853238 14572 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "268581e0456549218cd7fff0cb991913" member_type: VOTER last_known_addr { host: "127.13.13.126" port: 37371 } } peers { permanent_uuid: "dd4c83ae72e94b369d028ea29fc7e62c" member_type: VOTER last_known_addr { host: "127.13.13.125" port: 37877 } } peers { permanent_uuid: "3f97e33e85db48b3a34d544fa3495dd9" member_type: VOTER last_known_addr { host: "127.13.13.124" port: 46557 } } }
08:13:23.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.854049 14572 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: This master's current role is: FOLLOWER
08:13:23.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.854782 14562 sys_catalog.cc:564] T 00000000000000000000000000000000 P 3f97e33e85db48b3a34d544fa3495dd9 [sys.catalog]: configured and running, proceeding with master startup.
08:13:23.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:23.865466 13365 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 14461
08:13:24.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T08:13:24Z chronyd exiting
08:13:26.002 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:13:26.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:26.004037 12644 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 12938
08:13:26.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:26.025444 12644 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 13083
08:13:26.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:26.047694 12644 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 13222
08:13:26.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:26.069376 12644 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 12656
08:13:26.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:26.098152 12644 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 12726
08:13:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250625 08:13:26.122766 12644 external_mini_cluster.cc:1620] Killing /tmp/dist-test-taskzFzfv3/build/tsan/bin/kudu with pid 12801
08:13:27.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-06-25T08:13:27Z chronyd exiting
08:13:28.230 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires: failed attempt 1
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.13.13.125:37877,127.13.13.126:37371,127.13.13.124:46557). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386) ~[test/:?]
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40) ~[kudu-test-utils-1.18.0-SNAPSHOT-unshaded.jar:1.18.0-SNAPSHOT]
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158) ~[main/:?]
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: cannot re-acquire authentication token after 5 attempts (Couldn't find a valid master in (127.13.13.125:37877,127.13.13.126:37371,127.13.13.124:46557). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client does not have Kerberos credentials (tgt). Authentication tokens were not used because this connection will be used to acquire a new token and therefore requires primary credentials])
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity$3.get(TestSecurity.java:386)
at org.apache.kudu.test.junit.AssertHelpers.assertEventuallyTrue(AssertHelpers.java:40)
at org.apache.kudu.client.TestSecurity.assertEventualAuthenticationFailure(TestSecurity.java:378)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectExpires(TestSecurity.java:412)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:158)
at org.apache.kudu.client.AuthnTokenReacquirer$1NewAuthnTokenErrB.call(AuthnTokenReacquirer.java:141)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more