02:03:47.079 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
02:03:47.092 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4yviL0/test-tmp
02:03:47.099 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:03:47.102 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4yviL0/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:03:47.358 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
02:03:47.534 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:03:47.536 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:03:47.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:47.541239 22309 mini_oidc.cc:150] Starting JWKS server
02:03:47.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:47.545106 22309 webserver.cc:533] Webserver started at http://127.0.0.1:36439/ using document root <none> and password file <none>
02:03:47.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:47.545794 22309 mini_oidc.cc:186] Starting OIDC Discovery server
02:03:47.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:47.548358 22309 webserver.cc:533] Webserver started at http://127.0.0.1:39317/ using document root <none> and password file <none>
02:03:47.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T02:03:47Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:03:47.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T02:03:47Z Disabled control of system clock
02:03:47.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:47.610958 22309 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:03:47.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:03:47.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/wal
02:03:47.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/data
02:03:47.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:47.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:47.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:47.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:47.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:47.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:47.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/logs
02:03:47.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/data/info.pb
02:03:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:47.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:47.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:47.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:47.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:47.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:47.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:47.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.126:35723
02:03:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.126
02:03:47.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:47.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:03:47.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:47.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:47.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:47.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:47.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:03:47.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:47.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36439/jwks.json with env {}
02:03:47.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:47.921130 22321 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:47.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:47.921685 22321 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:47.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:47.922284 22321 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:47.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:47.955526 22321 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:03:47.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:47.955986 22321 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:47.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:47.956208 22321 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:47.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:47.956398 22321 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:03:47.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:47.956646 22321 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:47.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:47.956797 22321 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36439/jwks.json
02:03:47.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:47.994158 22321 master_runner.cc:387] Master server non-default flags:
02:03:47.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:47.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:03:47.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/data
02:03:47.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/wal
02:03:47.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:47.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:03:47.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:47.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:47.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:47.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.126:35723
02:03:47.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:47.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:47.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:47.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36439/jwks.json
02:03:47.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:47.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:47.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/data/info.pb
02:03:48.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.126
02:03:48.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:48.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:48.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:48.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:48.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:48.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:48.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/logs
02:03:48.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:48.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:48.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:48.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:48.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:48.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:03:48.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:48.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:03:48.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:03:48.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:48.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:47.995532 22321 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:48.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:47.997423 22321 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:48.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:48.010278 22327 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:49.414551 22326 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 22321
02:03:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:49.527177 22321 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.516s user 0.548s sys 0.948s
02:03:49.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:49.527691 22321 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.516s user 0.548s sys 0.948s
02:03:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:48.012097 22328 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:49.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:49.528964 22329 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1515 milliseconds
02:03:49.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.531409 22321 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:03:49.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:49.531505 22330 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.533926 22321 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:49.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.536983 22321 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:49.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.538443 22321 hybrid_clock.cc:648] HybridClock initialized: now 1781575429538403 us; error 55 us; skew 500 ppm
02:03:49.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.539464 22321 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:49.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.548156 22321 webserver.cc:533] Webserver started at http://127.21.201.126:35841/ using document root <none> and password file <none>
02:03:49.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.549414 22321 fs_manager.cc:362] Metadata directory not provided
02:03:49.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.549710 22321 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:49.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.550264 22321 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.555517 22321 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/data/instance:
02:03:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0e0774a9c3d14612b061347ecf23dc28"
02:03:49.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:49 on dist-test-slave-nkh1"
02:03:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.556933 22321 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/wal/instance:
02:03:49.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0e0774a9c3d14612b061347ecf23dc28"
02:03:49.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:49 on dist-test-slave-nkh1"
02:03:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.564256 22321 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
02:03:49.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.570111 22337 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:03:49.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.571518 22321 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.005s
02:03:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.571844 22321 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/wal
02:03:49.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0e0774a9c3d14612b061347ecf23dc28"
02:03:49.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:49 on dist-test-slave-nkh1"
02:03:49.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.572297 22321 fs_report.cc:389] FS layout report
02:03:49.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:49.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/wal
02:03:49.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/wal
02:03:49.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/data/data
02:03:49.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:49.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:49.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:49.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:49.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:49.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:49.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:49.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:49.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:49.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:49.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:49.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.634500 22321 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:03:49.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.636029 22321 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:49.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.636441 22321 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:49.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.709687 22321 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.126:35723
02:03:49.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.709793 22388 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.126:35723 every 8 connection(s)
02:03:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.712360 22321 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/data/info.pb
02:03:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.718226 22389 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:03:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.723006 22309 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 22321
02:03:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.723379 22309 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-0/wal/instance
02:03:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.726785 22309 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:03:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:03:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/wal
02:03:49.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/data
02:03:49.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:49.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:49.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:49.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:49.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:49.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:49.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/logs
02:03:49.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/data/info.pb
02:03:49.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:49.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:49.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.125:45319
02:03:49.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.125
02:03:49.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:49.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:03:49.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:49.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:49.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:49.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:49.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:03:49.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:49.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36439/jwks.json with env {}
02:03:49.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.730993 22389 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } has no permanent_uuid. Determining permanent_uuid...
02:03:49.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.763653 22389 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } has no permanent_uuid. Determining permanent_uuid...
02:03:49.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:49.766486 22338 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.201.125:45319: connect: Connection refused (error 111)
02:03:49.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:49.770473 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.125:45319: Network error: Client connection negotiation failed: client connection to 127.21.201.125:45319: connect: Connection refused (error 111)
02:03:49.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.819999 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } attempt: 1
02:03:49.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:49.824592 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.125:45319: Network error: Client connection negotiation failed: client connection to 127.21.201.125:45319: connect: Connection refused (error 111)
02:03:49.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.892985 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } attempt: 2
02:03:49.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:49.897611 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.125:45319: Network error: Client connection negotiation failed: client connection to 127.21.201.125:45319: connect: Connection refused (error 111)
02:03:49.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:49.988991 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } attempt: 3
02:03:49.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:49.993163 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.125:45319: Network error: Client connection negotiation failed: client connection to 127.21.201.125:45319: connect: Connection refused (error 111)
02:03:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.055135 22391 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:50.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.055691 22391 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:50.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.056218 22391 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.089900 22391 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:03:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.090197 22391 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.090394 22391 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.090580 22391 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:03:50.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.090759 22391 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:50.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.090898 22391 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36439/jwks.json
02:03:50.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:50.129276 22391 master_runner.cc:387] Master server non-default flags:
02:03:50.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:50.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:03:50.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:50.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/data
02:03:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/wal
02:03:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:03:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.125:45319
02:03:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36439/jwks.json
02:03:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/data/info.pb
02:03:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.125
02:03:50.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:50.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/logs
02:03:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:50.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:03:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:50.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:03:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:03:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:50.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:50.131065 22391 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:50.133493 22391 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:50.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:50.136555 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } attempt: 4
02:03:50.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.141666 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.125:45319: Network error: Client connection negotiation failed: client connection to 127.21.201.125:45319: connect: Connection refused (error 111)
02:03:50.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.146577 22400 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:50.441130 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } attempt: 5
02:03:50.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.449512 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.125:45319: Network error: Client connection negotiation failed: client connection to 127.21.201.125:45319: connect: Connection refused (error 111)
02:03:50.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:50.997136 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } attempt: 6
02:03:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.008500 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.125:45319: Network error: Client connection negotiation failed: client connection to 127.21.201.125:45319: connect: Connection refused (error 111)
02:03:51.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.149402 22401 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:51.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:50.153260 22403 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:51.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.381865 22402 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1229 milliseconds
02:03:51.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.381984 22391 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:03:51.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.383457 22391 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:51.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.386650 22391 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:51.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.388109 22391 hybrid_clock.cc:648] HybridClock initialized: now 1781575431388025 us; error 66 us; skew 500 ppm
02:03:51.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.388958 22391 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:51.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.395725 22391 webserver.cc:533] Webserver started at http://127.21.201.125:36015/ using document root <none> and password file <none>
02:03:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.396685 22391 fs_manager.cc:362] Metadata directory not provided
02:03:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.396871 22391 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:51.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.397464 22391 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:51.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.402504 22391 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/data/instance:
02:03:51.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67d8444db3fc4620a8aee54572d88506"
02:03:51.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:51 on dist-test-slave-nkh1"
02:03:51.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.403530 22391 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/wal/instance:
02:03:51.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67d8444db3fc4620a8aee54572d88506"
02:03:51.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:51 on dist-test-slave-nkh1"
02:03:51.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.410943 22391 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.004s
02:03:51.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.416898 22412 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:03:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.418109 22391 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
02:03:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.418411 22391 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/wal
02:03:51.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "67d8444db3fc4620a8aee54572d88506"
02:03:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:51 on dist-test-slave-nkh1"
02:03:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.418732 22391 fs_report.cc:389] FS layout report
02:03:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/wal
02:03:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/wal
02:03:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/data/data
02:03:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:51.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:51.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:51.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:51.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.466528 22391 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:03:51.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.468463 22391 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:51.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.469070 22391 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:51.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.550961 22391 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.125:45319
02:03:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.551067 22463 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.125:45319 every 8 connection(s)
02:03:51.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.554870 22391 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/data/info.pb
02:03:51.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.561210 22464 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:03:51.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.564535 22309 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 22391
02:03:51.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.565043 22309 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-1/wal/instance
02:03:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.567252 22309 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:03:51.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:03:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/wal
02:03:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/data
02:03:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:51.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/logs
02:03:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/data/info.pb
02:03:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:51.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:03:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:51.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.124:37613
02:03:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.124
02:03:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:03:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:51.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:51.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:03:51.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:51.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36439/jwks.json with env {}
02:03:51.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.576342 22464 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } has no permanent_uuid. Determining permanent_uuid...
02:03:51.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.600812 22464 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } has no permanent_uuid. Determining permanent_uuid...
02:03:51.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.617041 22464 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } has no permanent_uuid. Determining permanent_uuid...
02:03:51.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.620061 22415 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:51.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.625101 22464 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:51.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.674662 22464 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 1
02:03:51.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.679270 22464 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:51.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.747759 22464 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 2
02:03:51.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.752206 22464 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:51.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.843619 22464 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 3
02:03:51.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.848452 22464 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:51.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.890956 22466 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.891609 22466 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.892140 22466 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:51.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.926271 22466 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:03:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.926555 22466 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.926846 22466 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:03:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.927110 22466 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:03:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.927302 22466 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:03:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.927449 22466 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:36439/jwks.json
02:03:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.968781 22466 master_runner.cc:387] Master server non-default flags:
02:03:51.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:03:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/data
02:03:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/wal
02:03:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:03:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:03:51.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.124:37613
02:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36439/jwks.json
02:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:51.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/data/info.pb
02:03:51.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.124
02:03:51.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:51.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:51.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:51.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:51.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:51.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/logs
02:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:03:51.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:51.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:03:51.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:03:51.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:51.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.970219 22466 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:51.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.972564 22466 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:51.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.986613 22476 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:51.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:51.991845 22464 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 4
02:03:52.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.999949 22464 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:52.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:52.069092 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } attempt: 7
02:03:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:52.102469 22389 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } has no permanent_uuid. Determining permanent_uuid...
02:03:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:52.125051 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:52.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:52.183631 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 1
02:03:52.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:52.192636 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:52.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:52.274237 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 2
02:03:52.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:52.283223 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:52.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:52.299610 22464 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 5
02:03:52.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:52.309194 22464 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:52.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:52.368827 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 3
02:03:52.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:52.379601 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:52.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:52.520313 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 4
02:03:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:52.530893 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:52.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:52.814550 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 5
02:03:52.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:52.826115 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:52.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:52.856832 22464 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 6
02:03:52.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:52.885077 22464 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:53.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.378743 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 6
02:03:53.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:53.410648 22389 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.21.201.124:37613: Network error: Client connection negotiation failed: client connection to 127.21.201.124:37613: connect: Connection refused (error 111)
02:03:53.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:53.392022 22475 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 22466
02:03:53.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:53.708789 22475 kernel_stack_watchdog.cc:198] Thread 22466 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
02:03:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:03:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:03:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:03:53.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:03:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:51.988653 22477 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:53.715710 22466 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.721s user 0.635s sys 1.085s
02:03:53.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:53.716730 22466 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.729s user 0.636s sys 1.085s
02:03:53.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:53.716746 22478 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1720 milliseconds
02:03:53.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:53.717945 22479 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:53.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.717916 22466 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:03:53.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.721482 22466 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:53.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.723793 22466 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:53.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.725250 22466 hybrid_clock.cc:648] HybridClock initialized: now 1781575433725215 us; error 36 us; skew 500 ppm
02:03:53.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.726199 22466 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.732821 22466 webserver.cc:533] Webserver started at http://127.21.201.124:38657/ using document root <none> and password file <none>
02:03:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.733877 22466 fs_manager.cc:362] Metadata directory not provided
02:03:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.734133 22466 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.734658 22466 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:53.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.738979 22466 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/data/instance:
02:03:53.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0cbd322916948d186308bd4ddc4d6fb"
02:03:53.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:53 on dist-test-slave-nkh1"
02:03:53.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.740278 22466 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/wal/instance:
02:03:53.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0cbd322916948d186308bd4ddc4d6fb"
02:03:53.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:53 on dist-test-slave-nkh1"
02:03:53.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.747166 22466 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.005s
02:03:53.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.753016 22493 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:03:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.754087 22466 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
02:03:53.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.754616 22466 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/wal
02:03:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f0cbd322916948d186308bd4ddc4d6fb"
02:03:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:53 on dist-test-slave-nkh1"
02:03:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.754968 22466 fs_report.cc:389] FS layout report
02:03:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:53.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/wal
02:03:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/wal
02:03:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/data/data
02:03:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:53.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:53.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.810369 22466 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:03:53.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.812095 22466 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:53.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.812610 22466 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:53.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.889627 22466 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.124:37613
02:03:53.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.889803 22544 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.124:37613 every 8 connection(s)
02:03:53.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.892549 22466 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/data/info.pb
02:03:53.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.894335 22309 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 22466
02:03:53.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.894876 22309 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/master-2/wal/instance
02:03:53.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.900067 22545 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:03:53.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.917557 22545 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } has no permanent_uuid. Determining permanent_uuid...
02:03:53.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.945616 22464 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 7
02:03:53.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.955349 22545 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } has no permanent_uuid. Determining permanent_uuid...
02:03:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.971994 22545 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } has no permanent_uuid. Determining permanent_uuid...
02:03:53.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.974366 22464 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506: Bootstrap starting.
02:03:53.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.982839 22464 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506: Neither blocks nor log segments found. Creating new log.
02:03:53.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.985723 22464 log.cc:826] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506: Log is configured to *not* fsync() on all Append() calls
02:03:53.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.991487 22464 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506: No bootstrap required, opened a new log
02:03:53.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:53.991818 22545 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb: Bootstrap starting.
02:03:54.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.000819 22545 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb: Neither blocks nor log segments found. Creating new log.
02:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.003185 22545 log.cc:826] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb: Log is configured to *not* fsync() on all Append() calls
02:03:54.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.009428 22545 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb: No bootstrap required, opened a new log
02:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.021324 22464 raft_consensus.cc:359] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } }
02:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.022223 22464 raft_consensus.cc:385] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:03:54.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.022526 22464 raft_consensus.cc:740] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 67d8444db3fc4620a8aee54572d88506, State: Initialized, Role: FOLLOWER
02:03:54.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.023480 22464 consensus_queue.cc:260] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } }
02:03:54.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.026916 22558 sys_catalog.cc:455] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } } }
02:03:54.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.028057 22558 sys_catalog.cc:458] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [sys.catalog]: This master's current role is: FOLLOWER
02:03:54.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.029218 22464 sys_catalog.cc:565] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [sys.catalog]: configured and running, proceeding with master startup.
02:03:54.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.041774 22545 raft_consensus.cc:359] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } }
02:03:54.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.043252 22545 raft_consensus.cc:385] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:03:54.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.044436 22545 raft_consensus.cc:740] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f0cbd322916948d186308bd4ddc4d6fb, State: Initialized, Role: FOLLOWER
02:03:54.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.045563 22545 consensus_queue.cc:260] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } }
02:03:54.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.049129 22567 sys_catalog.cc:455] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } } }
02:03:54.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.050750 22567 sys_catalog.cc:458] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [sys.catalog]: This master's current role is: FOLLOWER
02:03:54.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.051756 22545 sys_catalog.cc:565] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [sys.catalog]: configured and running, proceeding with master startup.
02:03:54.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:54.068339 22499 tablet.cc:2406] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:03:54.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:54.068645 22570 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:54.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:54.069005 22570 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:54.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.057607 22391 master_runner.cc:187] Error getting master registration for 127.21.201.126:35723: OK, instance_id { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" instance_seqno: 1781575429689865 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:03:54.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.098945 22391 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:03:54.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:54.108217 22582 catalog_manager.cc:1593] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:03:54.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:54.109047 22582 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:03:54.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.092772 22466 master_runner.cc:187] Error getting master registration for 127.21.201.126:35723: OK, instance_id { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" instance_seqno: 1781575429689865 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:03:54.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.133761 22466 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:03:54.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:54.228593 22418 tablet.cc:2406] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:03:54.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.273665 22558 raft_consensus.cc:493] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:03:54.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.274214 22558 raft_consensus.cc:515] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } }
02:03:54.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.283735 22520 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "67d8444db3fc4620a8aee54572d88506" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f0cbd322916948d186308bd4ddc4d6fb" is_pre_election: true
02:03:54.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.283003 22558 leader_election.cc:290] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0e0774a9c3d14612b061347ecf23dc28 (127.21.201.126:35723), f0cbd322916948d186308bd4ddc4d6fb (127.21.201.124:37613)
02:03:54.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.284669 22520 raft_consensus.cc:2468] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 67d8444db3fc4620a8aee54572d88506 in term 0.
02:03:54.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.285971 22415 leader_election.cc:304] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 67d8444db3fc4620a8aee54572d88506, f0cbd322916948d186308bd4ddc4d6fb; no voters:
02:03:54.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.286949 22558 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [term 0 FOLLOWER]: Leader pre-election won for term 1
02:03:54.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.287292 22558 raft_consensus.cc:493] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:03:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.287817 22558 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [term 0 FOLLOWER]: Advancing to term 1
02:03:54.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.295504 22558 raft_consensus.cc:515] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } }
02:03:54.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.298899 22558 leader_election.cc:290] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [CANDIDATE]: Term 1 election: Requested vote from peers 0e0774a9c3d14612b061347ecf23dc28 (127.21.201.126:35723), f0cbd322916948d186308bd4ddc4d6fb (127.21.201.124:37613)
02:03:54.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.299791 22520 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "67d8444db3fc4620a8aee54572d88506" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f0cbd322916948d186308bd4ddc4d6fb"
02:03:54.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.300431 22520 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [term 0 FOLLOWER]: Advancing to term 1
02:03:54.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.310055 22520 raft_consensus.cc:2468] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 67d8444db3fc4620a8aee54572d88506 in term 1.
02:03:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.314335 22415 leader_election.cc:304] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 67d8444db3fc4620a8aee54572d88506, f0cbd322916948d186308bd4ddc4d6fb; no voters:
02:03:54.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.315465 22558 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [term 1 FOLLOWER]: Leader election won for term 1
02:03:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.318382 22558 raft_consensus.cc:697] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [term 1 LEADER]: Becoming Leader. State: Replica: 67d8444db3fc4620a8aee54572d88506, State: Running, Role: LEADER
02:03:54.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.320022 22558 consensus_queue.cc:237] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } }
02:03:54.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.344889 22584 sys_catalog.cc:455] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 67d8444db3fc4620a8aee54572d88506. Latest consensus state: current_term: 1 leader_uuid: "67d8444db3fc4620a8aee54572d88506" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } } }
02:03:54.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.345728 22584 sys_catalog.cc:458] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [sys.catalog]: This master's current role is: LEADER
02:03:54.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.350060 22586 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
02:03:54.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.358446 22586 catalog_manager.cc:1519] Initializing Kudu cluster ID...
02:03:54.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.384965 22520 raft_consensus.cc:1275] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [term 1 FOLLOWER]: Refusing update from remote peer 67d8444db3fc4620a8aee54572d88506: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:03:54.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.386824 22584 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:03:54.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.436156 22587 mvcc.cc:204] Tried to move back new op lower bound from 7297332979220807680 to 7297332978984038400. Current Snapshot: MvccSnapshot[applied={T|T < 7297332979220807680}]
02:03:54.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.439240 22567 sys_catalog.cc:455] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [sys.catalog]: SysCatalogTable state changed. Reason: New leader 67d8444db3fc4620a8aee54572d88506. Latest consensus state: current_term: 1 leader_uuid: "67d8444db3fc4620a8aee54572d88506" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } } }
02:03:54.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.442144 22567 sys_catalog.cc:458] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [sys.catalog]: This master's current role is: FOLLOWER
02:03:54.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.444155 22389 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } attempt: 7
02:03:54.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.481484 22567 sys_catalog.cc:455] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "67d8444db3fc4620a8aee54572d88506" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } } }
02:03:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.483717 22567 sys_catalog.cc:458] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb [sys.catalog]: This master's current role is: FOLLOWER
02:03:54.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.490875 22558 sys_catalog.cc:455] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "67d8444db3fc4620a8aee54572d88506" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } } }
02:03:54.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.491671 22558 sys_catalog.cc:458] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [sys.catalog]: This master's current role is: LEADER
02:03:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.500252 22586 catalog_manager.cc:1382] Generated new cluster ID: d89534fe74af4afe9ccc97c8fdb9af32
02:03:54.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.500604 22586 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
02:03:54.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.534305 22389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28: Bootstrap starting.
02:03:54.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.559360 22389 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28: Neither blocks nor log segments found. Creating new log.
02:03:54.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.564567 22389 log.cc:826] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28: Log is configured to *not* fsync() on all Append() calls
02:03:54.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.597613 22389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28: No bootstrap required, opened a new log
02:03:54.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.626034 22586 catalog_manager.cc:1405] Generated new certificate authority record
02:03:54.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.632999 22586 catalog_manager.cc:1539] Loading token signing keys...
02:03:54.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.716434 22389 raft_consensus.cc:359] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } }
02:03:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.717317 22389 raft_consensus.cc:385] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:03:54.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.719099 22389 raft_consensus.cc:740] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0e0774a9c3d14612b061347ecf23dc28, State: Initialized, Role: FOLLOWER
02:03:54.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.720240 22389 consensus_queue.cc:260] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } }
02:03:54.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.724961 22586 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506: Generated new TSK 0
02:03:54.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.728040 22586 catalog_manager.cc:1549] Initializing in-progress tserver states...
02:03:54.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.736959 22595 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } } }
02:03:54.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.737995 22595 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [sys.catalog]: This master's current role is: FOLLOWER
02:03:54.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.741003 22389 sys_catalog.cc:565] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [sys.catalog]: configured and running, proceeding with master startup.
02:03:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.742480 22363 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "67d8444db3fc4620a8aee54572d88506" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0e0774a9c3d14612b061347ecf23dc28"
02:03:54.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.743657 22363 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [term 0 FOLLOWER]: Advancing to term 1
02:03:54.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.752174 22364 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "67d8444db3fc4620a8aee54572d88506" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0e0774a9c3d14612b061347ecf23dc28" is_pre_election: true
02:03:54.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.758311 22309 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:03:54.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/wal
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/data
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/logs
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/data/info.pb
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:54.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:03:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.65:0
02:03:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.65
02:03:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.65
02:03:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:03:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:03:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:03:54.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.775605 22363 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 67d8444db3fc4620a8aee54572d88506 in term 1.
02:03:54.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.777828 22362 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [term 1 FOLLOWER]: Refusing update from remote peer 67d8444db3fc4620a8aee54572d88506: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:03:54.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.788007 22584 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:03:54.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.872865 22595 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 67d8444db3fc4620a8aee54572d88506. Latest consensus state: current_term: 1 leader_uuid: "67d8444db3fc4620a8aee54572d88506" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } } }
02:03:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.873593 22595 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [sys.catalog]: This master's current role is: FOLLOWER
02:03:54.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.873175 22604 sys_catalog.cc:455] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "67d8444db3fc4620a8aee54572d88506" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } } }
02:03:54.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.875697 22604 sys_catalog.cc:458] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28 [sys.catalog]: This master's current role is: FOLLOWER
02:03:54.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:54.903983 22612 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:03:55.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.116216 22582 catalog_manager.cc:1294] Loaded cluster ID: d89534fe74af4afe9ccc97c8fdb9af32
02:03:55.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.117091 22582 catalog_manager.cc:1587] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb: loading cluster ID for follower catalog manager: success
02:03:55.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.125801 22582 catalog_manager.cc:1609] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb: acquiring CA information for follower catalog manager: success
02:03:55.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.132498 22582 catalog_manager.cc:1637] T 00000000000000000000000000000000 P f0cbd322916948d186308bd4ddc4d6fb: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:03:55.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:55.272490 22597 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:55.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:55.273080 22597 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:55.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:55.273751 22597 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:55.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:55.308930 22597 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:55.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:55.309939 22597 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.201.65
02:03:55.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.349148 22597 tablet_server_runner.cc:78] Tablet server non-default flags:
02:03:55.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/data
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/wal
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.65:0
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/data/info.pb
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.65
02:03:55.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:03:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.65
02:03:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/logs
02:03:55.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:55.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:03:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:03:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:03:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:03:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.350900 22597 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:55.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.361331 22597 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:55.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.391185 22617 sys_catalog.cc:455] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "67d8444db3fc4620a8aee54572d88506" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0e0774a9c3d14612b061347ecf23dc28" member_type: VOTER last_known_addr { host: "127.21.201.126" port: 35723 } } peers { permanent_uuid: "67d8444db3fc4620a8aee54572d88506" member_type: VOTER last_known_addr { host: "127.21.201.125" port: 45319 } } peers { permanent_uuid: "f0cbd322916948d186308bd4ddc4d6fb" member_type: VOTER last_known_addr { host: "127.21.201.124" port: 37613 } } }
02:03:55.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.392831 22617 sys_catalog.cc:458] T 00000000000000000000000000000000 P 67d8444db3fc4620a8aee54572d88506 [sys.catalog]: This master's current role is: LEADER
02:03:55.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:55.404217 22621 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:55.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.929404 22612 catalog_manager.cc:1294] Loaded cluster ID: d89534fe74af4afe9ccc97c8fdb9af32
02:03:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.929700 22612 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28: loading cluster ID for follower catalog manager: success
02:03:55.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.962877 22612 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28: acquiring CA information for follower catalog manager: success
02:03:55.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:55.993710 22612 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 0e0774a9c3d14612b061347ecf23dc28: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:03:57.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:56.800026 22619 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 22597
02:03:57.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:57.814538 22597 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.408s user 0.856s sys 1.219s
02:03:57.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:57.815531 22597 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.410s user 0.856s sys 1.219s
02:03:57.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:55.406702 22622 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:57.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:57.830229 22631 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:57.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.831185 22597 server_base.cc:1061] running on GCE node
02:03:57.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.832995 22597 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:03:57.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.836082 22597 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:03:57.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.837534 22597 hybrid_clock.cc:648] HybridClock initialized: now 1781575437837404 us; error 130 us; skew 500 ppm
02:03:57.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.838583 22597 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:03:57.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.845968 22597 webserver.cc:533] Webserver started at http://127.21.201.65:36193/ using document root <none> and password file <none>
02:03:57.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.847146 22597 fs_manager.cc:362] Metadata directory not provided
02:03:57.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.847409 22597 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:03:57.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.847965 22597 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:03:57.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.854439 22597 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/data/instance:
02:03:57.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2be6fe97dac6425aaee261181aaa0ee3"
02:03:57.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:57 on dist-test-slave-nkh1"
02:03:57.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.855909 22597 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/wal/instance:
02:03:57.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2be6fe97dac6425aaee261181aaa0ee3"
02:03:57.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:57 on dist-test-slave-nkh1"
02:03:57.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.865989 22597 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.008s sys 0.000s
02:03:57.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.874681 22636 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:03:57.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.876559 22597 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.004s
02:03:57.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.877038 22597 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/wal
02:03:57.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2be6fe97dac6425aaee261181aaa0ee3"
02:03:57.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:03:57 on dist-test-slave-nkh1"
02:03:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.877509 22597 fs_report.cc:389] FS layout report
02:03:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:03:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/wal
02:03:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/wal
02:03:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/data/data
02:03:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:03:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:03:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:03:57.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:03:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:03:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:03:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:03:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:03:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:03:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:03:57.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:03:57.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.988346 22597 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:03:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.990685 22597 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:03:57.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.991232 22597 kserver.cc:163] Server-wide thread pool size limit: 3276
02:03:57.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:57.994809 22597 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:03:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.000708 22597 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:03:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.000989 22597 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:03:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.001271 22597 ts_tablet_manager.cc:616] Registered 0 tablets
02:03:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.001480 22597 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:03:58.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.204877 22597 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.65:34791
02:03:58.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.205004 22749 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.65:34791 every 8 connection(s)
02:03:58.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.208551 22597 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/data/info.pb
02:03:58.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.211122 22309 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 22597
02:03:58.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.211750 22309 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-0/wal/instance
02:03:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.236778 22309 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:03:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:03:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/wal
02:03:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/data
02:03:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:03:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:03:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:03:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:58.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/logs
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/data/info.pb
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.66:0
02:03:58.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.66
02:03:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.66
02:03:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:03:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:03:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:58.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:03:58.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.326922 22750 heartbeater.cc:344] Connected to a master server at 127.21.201.124:37613
02:03:58.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.328425 22750 heartbeater.cc:461] Registering TS with master...
02:03:58.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.330215 22750 heartbeater.cc:507] Master 127.21.201.124:37613 requested a full tablet report, sending...
02:03:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.333830 22510 ts_manager.cc:194] Registered new tserver with Master: 2be6fe97dac6425aaee261181aaa0ee3 (127.21.201.65:34791)
02:03:58.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.340762 22754 heartbeater.cc:344] Connected to a master server at 127.21.201.125:45319
02:03:58.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.341132 22754 heartbeater.cc:461] Registering TS with master...
02:03:58.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.341804 22757 heartbeater.cc:344] Connected to a master server at 127.21.201.126:35723
02:03:58.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.342005 22754 heartbeater.cc:507] Master 127.21.201.125:45319 requested a full tablet report, sending...
02:03:58.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.342274 22757 heartbeater.cc:461] Registering TS with master...
02:03:58.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.343132 22757 heartbeater.cc:507] Master 127.21.201.126:35723 requested a full tablet report, sending...
02:03:58.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.345716 22429 ts_manager.cc:194] Registered new tserver with Master: 2be6fe97dac6425aaee261181aaa0ee3 (127.21.201.65:34791)
02:03:58.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.350224 22354 ts_manager.cc:194] Registered new tserver with Master: 2be6fe97dac6425aaee261181aaa0ee3 (127.21.201.65:34791)
02:03:58.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.351415 22429 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.201.65:46877
02:03:58.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:58.631542 22751 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:03:58.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:58.632081 22751 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:03:58.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:58.632488 22751 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:03:58.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:58.667605 22751 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:03:58.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:58.668479 22751 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.201.66
02:03:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.704738 22751 tablet_server_runner.cc:78] Tablet server non-default flags:
02:03:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:03:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:03:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:03:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:03:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/data
02:03:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/wal
02:03:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.66:0
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/data/info.pb
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.66
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.66
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/logs
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:03:58.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:03:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:03:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:03:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:03:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:03:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:03:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:03:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.706157 22751 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:03:58.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:58.708354 22751 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:03:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:58.735846 22766 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:03:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:03:59.355762 22754 heartbeater.cc:499] Master 127.21.201.125:45319 was elected leader, sending a full tablet report...
02:04:00.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:00.132892 22765 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 22751
02:04:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:00.619027 22751 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.881s user 0.632s sys 1.120s
02:04:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:00.619401 22751 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.882s user 0.632s sys 1.120s
02:04:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:03:58.737879 22767 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:00.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:00.622630 22769 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:00.626575 22768 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1874 milliseconds
02:04:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.626734 22751 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.628230 22751 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.630380 22751 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.631902 22751 hybrid_clock.cc:648] HybridClock initialized: now 1781575440631832 us; error 50 us; skew 500 ppm
02:04:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.632973 22751 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:00.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.639372 22751 webserver.cc:533] Webserver started at http://127.21.201.66:36487/ using document root <none> and password file <none>
02:04:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.640398 22751 fs_manager.cc:362] Metadata directory not provided
02:04:00.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.640604 22751 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:00.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.641052 22751 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:00.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.645897 22751 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/data/instance:
02:04:00.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7230faabe8624865addea8abecc15b85"
02:04:00.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:00 on dist-test-slave-nkh1"
02:04:00.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.647069 22751 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/wal/instance:
02:04:00.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7230faabe8624865addea8abecc15b85"
02:04:00.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:00 on dist-test-slave-nkh1"
02:04:00.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.655418 22751 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.009s sys 0.001s
02:04:00.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.661871 22777 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:00.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.663017 22751 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
02:04:00.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.663317 22751 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/wal
02:04:00.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7230faabe8624865addea8abecc15b85"
02:04:00.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:00 on dist-test-slave-nkh1"
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.663640 22751 fs_report.cc:389] FS layout report
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/wal
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/wal
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/data/data
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:00.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.727083 22751 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.728708 22751 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:00.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.729130 22751 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:00.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.732555 22751 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.736853 22751 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.737067 22751 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.737272 22751 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:00.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.737409 22751 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:04:00.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.891396 22751 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.66:43793
02:04:00.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.891598 22889 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.66:43793 every 8 connection(s)
02:04:00.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.894867 22751 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/data/info.pb
02:04:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.904328 22309 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 22751
02:04:00.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.904896 22309 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-1/wal/instance
02:04:00.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.919623 22309 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:00.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:00.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/wal
02:04:00.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/data
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/logs
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/data/info.pb
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:00.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.67:0
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.67
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.67
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
02:04:00.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.972715 22890 heartbeater.cc:344] Connected to a master server at 127.21.201.124:37613
02:04:00.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.973254 22890 heartbeater.cc:461] Registering TS with master...
02:04:00.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.976083 22890 heartbeater.cc:507] Master 127.21.201.124:37613 requested a full tablet report, sending...
02:04:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.979307 22510 ts_manager.cc:194] Registered new tserver with Master: 7230faabe8624865addea8abecc15b85 (127.21.201.66:43793)
02:04:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.979908 22891 heartbeater.cc:344] Connected to a master server at 127.21.201.125:45319
02:04:00.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.980244 22891 heartbeater.cc:461] Registering TS with master...
02:04:00.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.981105 22891 heartbeater.cc:507] Master 127.21.201.125:45319 requested a full tablet report, sending...
02:04:00.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.981683 22892 heartbeater.cc:344] Connected to a master server at 127.21.201.126:35723
02:04:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.982031 22892 heartbeater.cc:461] Registering TS with master...
02:04:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.982924 22892 heartbeater.cc:507] Master 127.21.201.126:35723 requested a full tablet report, sending...
02:04:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.983604 22429 ts_manager.cc:194] Registered new tserver with Master: 7230faabe8624865addea8abecc15b85 (127.21.201.66:43793)
02:04:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.985752 22354 ts_manager.cc:194] Registered new tserver with Master: 7230faabe8624865addea8abecc15b85 (127.21.201.66:43793)
02:04:00.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:00.985882 22429 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.201.66:33913
02:04:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:01.282814 22898 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:01.283294 22898 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:01.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:01.283695 22898 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:01.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:01.317005 22898 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:01.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:01.317795 22898 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.21.201.67
02:04:01.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:01.355373 22898 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.21.201.84:44973
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/data
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/wal
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.21.201.67:0
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/data/info.pb
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.21.201.67
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.21.201.126:35723,127.21.201.125:45319,127.21.201.124:37613
02:04:01.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.21.201.67
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/logs
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:01.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:01.356655 22898 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:01.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:01.358309 22898 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:01.371956 22906 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:01.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:01.989746 22891 heartbeater.cc:499] Master 127.21.201.125:45319 was elected leader, sending a full tablet report...
02:04:02.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:02.774489 22905 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 22898
02:04:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:01.374070 22907 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:02.815676 22898 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.443s user 0.471s sys 0.960s
02:04:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:02.816097 22898 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.444s user 0.471s sys 0.960s
02:04:02.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:02.818364 22909 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:02.820493 22908 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1443 milliseconds
02:04:02.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.820528 22898 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.821645 22898 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:02.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.823614 22898 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.824935 22898 hybrid_clock.cc:648] HybridClock initialized: now 1781575442824970 us; error 107 us; skew 500 ppm
02:04:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.825692 22898 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:02.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.831543 22898 webserver.cc:533] Webserver started at http://127.21.201.67:44047/ using document root <none> and password file <none>
02:04:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.832475 22898 fs_manager.cc:362] Metadata directory not provided
02:04:02.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.832679 22898 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:02.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.833055 22898 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:02.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.837419 22898 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/data/instance:
02:04:02.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e2912f824f647ebbab2973d848d28e6"
02:04:02.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:02 on dist-test-slave-nkh1"
02:04:02.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.838428 22898 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/wal/instance:
02:04:02.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e2912f824f647ebbab2973d848d28e6"
02:04:02.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:02 on dist-test-slave-nkh1"
02:04:02.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.845454 22898 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.005s
02:04:02.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.850800 22921 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:02.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.851903 22898 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
02:04:02.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.852202 22898 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/wal
02:04:02.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5e2912f824f647ebbab2973d848d28e6"
02:04:02.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:02 on dist-test-slave-nkh1"
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.852487 22898 fs_report.cc:389] FS layout report
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/wal
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/wal
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/data/data
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:02.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.913154 22898 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:02.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.914541 22898 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:02.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.914947 22898 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:02.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.917567 22898 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:02.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.921650 22898 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:02.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.921840 22898 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:04:02.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.922134 22898 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:02.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:02.922274 22898 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:04:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.076234 22898 rpc_server.cc:307] RPC server started. Bound to: 127.21.201.67:39445
02:04:03.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.076330 23033 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.21.201.67:39445 every 8 connection(s)
02:04:03.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.078848 22898 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/data/info.pb
02:04:03.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.084000 22309 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 22898
02:04:03.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.084489 22309 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster15324772075125506588/ts-2/wal/instance
02:04:03.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.117671 23034 heartbeater.cc:344] Connected to a master server at 127.21.201.124:37613
02:04:03.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.118171 23034 heartbeater.cc:461] Registering TS with master...
02:04:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.119521 23034 heartbeater.cc:507] Master 127.21.201.124:37613 requested a full tablet report, sending...
02:04:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.120366 23035 heartbeater.cc:344] Connected to a master server at 127.21.201.125:45319
02:04:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.120746 23035 heartbeater.cc:461] Registering TS with master...
02:04:03.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.121577 23035 heartbeater.cc:507] Master 127.21.201.125:45319 requested a full tablet report, sending...
02:04:03.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.122967 22510 ts_manager.cc:194] Registered new tserver with Master: 5e2912f824f647ebbab2973d848d28e6 (127.21.201.67:39445)
02:04:03.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.123139 23036 heartbeater.cc:344] Connected to a master server at 127.21.201.126:35723
02:04:03.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.123489 23036 heartbeater.cc:461] Registering TS with master...
02:04:03.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.123916 22429 ts_manager.cc:194] Registered new tserver with Master: 5e2912f824f647ebbab2973d848d28e6 (127.21.201.67:39445)
02:04:03.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.124810 23036 heartbeater.cc:507] Master 127.21.201.126:35723 requested a full tablet report, sending...
02:04:03.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.125593 22429 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.21.201.67:55321
02:04:03.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.127141 22354 ts_manager.cc:194] Registered new tserver with Master: 5e2912f824f647ebbab2973d848d28e6 (127.21.201.67:39445)
02:04:03.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.137482 22309 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
02:04:03.149 [DEBUG - main] (MiniKuduCluster.java:195) Response:
02:04:03.151 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
02:04:03.160 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.21.201.126"
port: 35723
}
bound_http_address {
host: "127.21.201.126"
port: 35841
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.21.201.125"
port: 45319
}
bound_http_address {
host: "127.21.201.125"
port: 36015
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.21.201.124"
port: 37613
}
bound_http_address {
host: "127.21.201.124"
port: 38657
}
}
}
02:04:03.163 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:04:03.166 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.21.201.65"
port: 34791
}
bound_http_address {
host: "127.21.201.65"
port: 36193
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.21.201.66"
port: 43793
}
bound_http_address {
host: "127.21.201.66"
port: 36487
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.21.201.67"
port: 39445
}
bound_http_address {
host: "127.21.201.67"
port: 44047
}
}
}
02:04:03.166 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
02:04:03.332 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
02:04:03.386 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
02:04:03.480 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-task4yviL0/test-tmp
02:04:03.481 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
02:04:03.481 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-task4yviL0/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
02:04:03.486 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
02:04:03.855 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:03.855 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
02:04:03.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.858891 23046 mini_oidc.cc:150] Starting JWKS server
02:04:03.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.862243 23046 webserver.cc:533] Webserver started at http://127.0.0.1:37909/ using document root <none> and password file <none>
02:04:03.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.862679 23046 mini_oidc.cc:186] Starting OIDC Discovery server
02:04:03.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:03.864233 23046 webserver.cc:533] Webserver started at http://127.0.0.1:38043/ using document root <none> and password file <none>
02:04:03.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
02:04:03.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/principal' for realm 'KRBTEST.COM',
02:04:03.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
02:04:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:03 dist-test-slave-nkh1 krb5kdc[23058](info): setting up network...
02:04:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
02:04:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:03 dist-test-slave-nkh1 krb5kdc[23058](info): set up 2 sockets
02:04:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:03 dist-test-slave-nkh1 krb5kdc[23058](info): commencing operation
02:04:03.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
02:04:04.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:04.129261 23035 heartbeater.cc:499] Master 127.21.201.125:45319 was elected leader, sending a full tablet report...
02:04:06.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.085928 23046 mini_kdc.cc:121] Time spent starting KDC: real 2.177s user 0.006s sys 0.014s
02:04:06.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
02:04:06.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:04:06.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
02:04:06.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
02:04:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:04:06.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
02:04:06.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
02:04:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:04:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
02:04:06.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
02:04:06.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/test-user.keytab.
02:04:06.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/test-user.keytab.
02:04:06.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:06 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575446, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:06.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:04:06.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T02:04:06Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
02:04:06.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T02:04:06Z Disabled control of system clock
02:04:06.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.129.190@KRBTEST.COM; defaulting to no policy
02:04:06.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:06.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.129.190@KRBTEST.COM" created.
02:04:06.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:06.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:06.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:06.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.129.190@KRBTEST.COM; defaulting to no policy
02:04:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:06.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.129.190@KRBTEST.COM" created.
02:04:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:06.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:06.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:06.312734 23046 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:06.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:06.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal
02:04:06.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data
02:04:06.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:06.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/logs
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data/info.pb
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.190:42071
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.190
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.190
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:06.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:06.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.452533 22746 debug-util.cc:398] Leaking SignalData structure 0x7b08000adda0 after lost signal to thread 22613
02:04:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.453514 22746 debug-util.cc:398] Leaking SignalData structure 0x7b08000adc40 after lost signal to thread 22749
02:04:06.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.641552 23076 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:06.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.642655 23076 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:06.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.643253 23076 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.678050 23076 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.678292 23076 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.678565 23076 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:06.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.678748 23076 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.678968 23076 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.679165 23076 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:06.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.679312 23076 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37909/jwks.json
02:04:06.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:06.721058 23076 master_runner.cc:387] Master server non-default flags:
02:04:06.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:06.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:06.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:06.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:06.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data
02:04:06.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal
02:04:06.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:06.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:06.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:06.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:06.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.190
02:04:06.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:06.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.190:42071
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data/info.pb
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.190
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:06.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/logs
02:04:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:06.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:04:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:04:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:04:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:06.722713 23076 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:06.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:06.724144 23076 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:06.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.739569 23082 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:08.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:08.141623 23081 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 23076
02:04:08.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:08.783002 23081 kernel_stack_watchdog.cc:198] Thread 23076 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 394ms:
02:04:08.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:04:08.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:04:08.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:04:08.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:04:08.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:08.783809 23076 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.039s user 0.000s sys 0.001s
02:04:08.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:08.784256 23076 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.040s user 0.000s sys 0.002s
02:04:08.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:08.786240 23086 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:08.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:06.743005 23083 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:08.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:08.803649 23084 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2060 milliseconds
02:04:08.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.803712 23076 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:08.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.805403 23076 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:08.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.808588 23076 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:08.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.810248 23076 hybrid_clock.cc:648] HybridClock initialized: now 1781575448810128 us; error 139 us; skew 500 ppm
02:04:08.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.811229 23076 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:08.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:08 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575448, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:08.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.818023 23076 init.cc:377] Logged in from keytab as kudu/127.22.129.190@KRBTEST.COM (short username kudu)
02:04:08.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.823972 23076 webserver.cc:533] Webserver started at http://127.22.129.190:42433/ using document root <none> and password file <none>
02:04:08.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.825230 23076 fs_manager.cc:362] Metadata directory not provided
02:04:08.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.825541 23076 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:08.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.826102 23076 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:08.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.832940 23076 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data/instance:
02:04:08.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "149cd51be90048ca9a2b819d788b874a"
02:04:08.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:08 on dist-test-slave-nkh1"
02:04:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.834491 23076 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal/instance:
02:04:08.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "149cd51be90048ca9a2b819d788b874a"
02:04:08.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:08 on dist-test-slave-nkh1"
02:04:08.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.844412 23076 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
02:04:08.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.852587 23096 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:08.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.854179 23076 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
02:04:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.854704 23076 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal
02:04:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "149cd51be90048ca9a2b819d788b874a"
02:04:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:08 on dist-test-slave-nkh1"
02:04:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.855150 23076 fs_report.cc:389] FS layout report
02:04:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal
02:04:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data/data
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.961359 23076 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.967051 23076 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:08.967505 23076 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.063629 23076 rpc_server.cc:307] RPC server started. Bound to: 127.22.129.190:42071
02:04:09.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.063733 23147 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.129.190:42071 every 8 connection(s)
02:04:09.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.066892 23076 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data/info.pb
02:04:09.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.073938 23148 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:04:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.076650 23046 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 23076
02:04:09.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.077194 23046 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal/instance
02:04:09.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.090534 23148 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } has no permanent_uuid. Determining permanent_uuid...
02:04:09.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.129.189@KRBTEST.COM; defaulting to no policy
02:04:09.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:09.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.129.189@KRBTEST.COM" created.
02:04:09.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:09 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575448, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.190@KRBTEST.COM for kudu/127.22.129.190@KRBTEST.COM
02:04:09.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.134177 23148 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } has no permanent_uuid. Determining permanent_uuid...
02:04:09.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.137288 23099 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.129.189:35809: connect: Connection refused (error 111)
02:04:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:09.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:09.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.142714 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.189:35809: Network error: Client connection negotiation failed: client connection to 127.22.129.189:35809: connect: Connection refused (error 111)
02:04:09.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.129.189@KRBTEST.COM; defaulting to no policy
02:04:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.129.189@KRBTEST.COM" created.
02:04:09.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:09.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:09.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:09.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.192265 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } attempt: 1
02:04:09.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.195261 23046 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/logs
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data/info.pb
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:09.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.189:35809
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.189
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.189
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.197757 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.189:35809: Network error: Client connection negotiation failed: client connection to 127.22.129.189:35809: connect: Connection refused (error 111)
02:04:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.266359 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } attempt: 2
02:04:09.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.271498 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.189:35809: Network error: Client connection negotiation failed: client connection to 127.22.129.189:35809: connect: Connection refused (error 111)
02:04:09.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.362946 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } attempt: 3
02:04:09.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.367955 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.189:35809: Network error: Client connection negotiation failed: client connection to 127.22.129.189:35809: connect: Connection refused (error 111)
02:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.511369 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } attempt: 4
02:04:09.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.515674 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.189:35809: Network error: Client connection negotiation failed: client connection to 127.22.129.189:35809: connect: Connection refused (error 111)
02:04:09.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.573257 23157 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:09.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.573792 23157 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:09.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.574352 23157 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.608170 23157 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.608512 23157 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:09.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.608798 23157 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:09.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.608995 23157 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:09.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.609201 23157 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:09.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.609391 23157 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:09.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.609548 23157 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37909/jwks.json
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.650043 23157 master_runner.cc:387] Master server non-default flags:
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:09.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.189
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.189:35809
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data/info.pb
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.189
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/logs
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:04:09.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:09.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.651571 23157 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:09.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.652725 23157 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:09.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.666893 23163 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:09.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:09.815264 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } attempt: 5
02:04:09.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.824908 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.189:35809: Network error: Client connection negotiation failed: client connection to 127.22.129.189:35809: connect: Connection refused (error 111)
02:04:10.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:10.372506 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } attempt: 6
02:04:10.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:10.401338 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.189:35809: Network error: Client connection negotiation failed: client connection to 127.22.129.189:35809: connect: Connection refused (error 111)
02:04:11.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.461905 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } attempt: 7
02:04:11.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:11.524169 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.189:35809: Network error: Client connection negotiation failed: client connection to 127.22.129.189:35809: connect: Connection refused (error 111)
02:04:11.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:11.068909 23162 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 23157
02:04:11.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:11.660151 23162 kernel_stack_watchdog.cc:198] Thread 23157 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
02:04:11.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:04:11.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:04:11.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:04:11.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:04:11.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:09.667172 23164 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:11.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:11.661476 23157 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.996s user 0.722s sys 1.259s
02:04:11.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:11.661855 23157 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.997s user 0.722s sys 1.260s
02:04:11.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:11.664103 23166 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:11.665709 23165 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1996 milliseconds
02:04:11.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.665791 23157 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:11.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.666990 23157 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:11.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.669572 23157 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:11.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.670907 23157 hybrid_clock.cc:648] HybridClock initialized: now 1781575451670878 us; error 64 us; skew 500 ppm
02:04:11.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.672039 23157 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:11.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:11 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575451, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:11.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.678571 23157 init.cc:377] Logged in from keytab as kudu/127.22.129.189@KRBTEST.COM (short username kudu)
02:04:11.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.684413 23157 webserver.cc:533] Webserver started at http://127.22.129.189:36035/ using document root <none> and password file <none>
02:04:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.685458 23157 fs_manager.cc:362] Metadata directory not provided
02:04:11.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.685729 23157 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:11.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.686305 23157 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:11.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.691167 23157 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data/instance:
02:04:11.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d31c363790e4b87aee262915bc24553"
02:04:11.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:11 on dist-test-slave-nkh1"
02:04:11.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.692391 23157 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal/instance:
02:04:11.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d31c363790e4b87aee262915bc24553"
02:04:11.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:11 on dist-test-slave-nkh1"
02:04:11.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.699620 23157 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
02:04:11.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.705600 23180 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:11.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.706845 23157 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
02:04:11.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.707288 23157 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d31c363790e4b87aee262915bc24553"
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:11 on dist-test-slave-nkh1"
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.707708 23157 fs_report.cc:389] FS layout report
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data/data
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:11.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:11.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:11.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.771485 23157 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:11.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.776922 23157 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:11.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.777439 23157 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.854683 23157 rpc_server.cc:307] RPC server started. Bound to: 127.22.129.189:35809
02:04:11.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.854780 23231 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.129.189:35809 every 8 connection(s)
02:04:11.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.857559 23157 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data/info.pb
02:04:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.860304 23046 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 23157
02:04:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.860877 23046 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal/instance
02:04:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.864895 23232 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:04:11.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.129.188@KRBTEST.COM; defaulting to no policy
02:04:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.129.188@KRBTEST.COM" created.
02:04:11.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.878765 23232 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } has no permanent_uuid. Determining permanent_uuid...
02:04:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:11 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575451, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.189@KRBTEST.COM for kudu/127.22.129.190@KRBTEST.COM
02:04:11.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:11.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:11.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:11.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.925581 23232 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } has no permanent_uuid. Determining permanent_uuid...
02:04:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.129.188@KRBTEST.COM; defaulting to no policy
02:04:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.129.188@KRBTEST.COM" created.
02:04:11.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:11 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575451, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.189@KRBTEST.COM for kudu/127.22.129.189@KRBTEST.COM
02:04:11.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.952277 23232 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } has no permanent_uuid. Determining permanent_uuid...
02:04:11.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:11.955060 23181 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:11.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:11.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:11.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:11.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:11.959475 23232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:11.961349 23046 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal
02:04:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data
02:04:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:11.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/logs
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data/info.pb
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.188:40757
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.188
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.188
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:11.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:12.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:12.009099 23232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } attempt: 1
02:04:12.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.013625 23232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:12.081985 23232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } attempt: 2
02:04:12.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.087970 23232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:12.179450 23232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } attempt: 3
02:04:12.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.184103 23232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.295099 23242 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:12.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.295640 23242 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:12.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.296262 23242 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:12.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:12.327605 23232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } attempt: 4
02:04:12.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.330559 23242 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.330896 23242 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.331358 23242 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.331670 23242 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:12.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.332042 23242 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:12.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.332552 23242 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:12.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.332971 23242 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37909/jwks.json
02:04:12.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.333559 23232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:12.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:12.373339 23242 master_runner.cc:387] Master server non-default flags:
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.188
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.188:40757
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data/info.pb
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.188
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:12.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/logs
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:12.374929 23242 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:12.376370 23242 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:12.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.390626 23249 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:12.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:12.633003 23232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } attempt: 5
02:04:12.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.640558 23232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:13.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.188117 23232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } attempt: 6
02:04:13.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:13.203729 23232 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:13.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.614711 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } attempt: 8
02:04:13.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:13 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575448, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.190@KRBTEST.COM for kudu/127.22.129.189@KRBTEST.COM
02:04:13.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.642786 23148 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } has no permanent_uuid. Determining permanent_uuid...
02:04:13.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:13.649870 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:13.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.715276 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } attempt: 1
02:04:13.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:13.721812 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:13.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.775319 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } attempt: 2
02:04:13.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:12.392990 23250 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:13.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:13.779103 23251 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1385 milliseconds
02:04:13.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.779315 23242 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:13.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:13.779135 23252 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:13.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:13.781234 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:13.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.784636 23242 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:13.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.787639 23242 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:13.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.789242 23242 hybrid_clock.cc:648] HybridClock initialized: now 1781575453789109 us; error 135 us; skew 500 ppm
02:04:13.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.790047 23242 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:13.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:13 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575453, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:13.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.796053 23242 init.cc:377] Logged in from keytab as kudu/127.22.129.188@KRBTEST.COM (short username kudu)
02:04:13.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.801780 23242 webserver.cc:533] Webserver started at http://127.22.129.188:43773/ using document root <none> and password file <none>
02:04:13.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.802870 23242 fs_manager.cc:362] Metadata directory not provided
02:04:13.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.803119 23242 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:13.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.803622 23242 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:13.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.808282 23242 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data/instance:
02:04:13.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "55320708ab29420da15b0cb01d4b79f3"
02:04:13.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:13 on dist-test-slave-nkh1"
02:04:13.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.809396 23242 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal/instance:
02:04:13.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "55320708ab29420da15b0cb01d4b79f3"
02:04:13.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:13 on dist-test-slave-nkh1"
02:04:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.817082 23242 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.005s
02:04:13.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.822710 23266 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:13.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.824009 23242 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
02:04:13.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.824446 23242 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal
02:04:13.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "55320708ab29420da15b0cb01d4b79f3"
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:13 on dist-test-slave-nkh1"
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.824847 23242 fs_report.cc:389] FS layout report
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data/data
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.857583 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } attempt: 3
02:04:13.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:13.862627 23148 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:13.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.888680 23242 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:13.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.895262 23242 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:13.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.895780 23242 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:13.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.977244 23242 rpc_server.cc:307] RPC server started. Bound to: 127.22.129.188:40757
02:04:13.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.977339 23317 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.129.188:40757 every 8 connection(s)
02:04:13.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.980163 23242 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data/info.pb
02:04:13.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.987041 23318 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
02:04:13.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.989356 23046 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 23242
02:04:13.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:13.989974 23046 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal/instance
02:04:14.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.002559 23318 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } has no permanent_uuid. Determining permanent_uuid...
02:04:14.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.018100 23148 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } attempt: 4
02:04:14.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:14 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575446, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.129.190@KRBTEST.COM
02:04:14.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:14 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575453, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.188@KRBTEST.COM for kudu/127.22.129.190@KRBTEST.COM
02:04:14.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:14 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575448, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.190@KRBTEST.COM for kudu/127.22.129.188@KRBTEST.COM
02:04:14.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.047127 23318 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } has no permanent_uuid. Determining permanent_uuid...
02:04:14.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:14 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575446, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.129.189@KRBTEST.COM
02:04:14.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.063701 23148 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: Bootstrap starting.
02:04:14.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:14 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575453, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.188@KRBTEST.COM for kudu/127.22.129.189@KRBTEST.COM
02:04:14.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.082432 23318 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } has no permanent_uuid. Determining permanent_uuid...
02:04:14.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.084625 23148 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: Neither blocks nor log segments found. Creating new log.
02:04:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.095100 23148 log.cc:826] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: Log is configured to *not* fsync() on all Append() calls
02:04:14.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:14 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575446, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.22.129.188@KRBTEST.COM
02:04:14.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:14 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575453, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.188@KRBTEST.COM for kudu/127.22.129.188@KRBTEST.COM
02:04:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.107959 23148 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: No bootstrap required, opened a new log
02:04:14.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.133185 23318 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: Bootstrap starting.
02:04:14.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.142814 23318 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: Neither blocks nor log segments found. Creating new log.
02:04:14.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.145545 23318 log.cc:826] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: Log is configured to *not* fsync() on all Append() calls
02:04:14.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.151196 23148 raft_consensus.cc:359] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:14.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.152087 23148 raft_consensus.cc:385] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:04:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.152550 23148 raft_consensus.cc:740] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 149cd51be90048ca9a2b819d788b874a, State: Initialized, Role: FOLLOWER
02:04:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.152810 23318 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: No bootstrap required, opened a new log
02:04:14.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.153532 23148 consensus_queue.cc:260] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:14.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.161755 23329 sys_catalog.cc:455] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:14.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.162580 23329 sys_catalog.cc:458] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.163983 23148 sys_catalog.cc:565] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: configured and running, proceeding with master startup.
02:04:14.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.180938 23318 raft_consensus.cc:359] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:14.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.182204 23318 raft_consensus.cc:385] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:04:14.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.182493 23318 raft_consensus.cc:740] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 55320708ab29420da15b0cb01d4b79f3, State: Initialized, Role: FOLLOWER
02:04:14.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.183516 23318 consensus_queue.cc:260] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:14.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.187065 23335 sys_catalog.cc:455] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:14.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.188126 23335 sys_catalog.cc:458] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.189244 23318 sys_catalog.cc:565] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: configured and running, proceeding with master startup.
02:04:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:14.233453 23102 tablet.cc:2406] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:04:14.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:14.228722 23342 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:04:14.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:14.236248 23342 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:04:14.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.213683 23076 master_runner.cc:187] Error getting master registration for 127.22.129.189:35809: OK, instance_id { permanent_uuid: "3d31c363790e4b87aee262915bc24553" instance_seqno: 1781575451833768 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:04:14.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:14.257170 23353 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
02:04:14.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.257498 23076 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:04:14.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:14.257704 23353 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
02:04:14.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.277387 23232 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } attempt: 7
02:04:14.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.254273 23242 master_runner.cc:187] Error getting master registration for 127.22.129.189:35809: OK, instance_id { permanent_uuid: "3d31c363790e4b87aee262915bc24553" instance_seqno: 1781575451833768 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
02:04:14.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.281213 23242 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:04:14.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:14 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575451, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.189@KRBTEST.COM for kudu/127.22.129.188@KRBTEST.COM
02:04:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.331067 23232 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: Bootstrap starting.
02:04:14.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.339833 23232 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: Neither blocks nor log segments found. Creating new log.
02:04:14.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.343147 23232 log.cc:826] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: Log is configured to *not* fsync() on all Append() calls
02:04:14.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.356549 23232 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: No bootstrap required, opened a new log
02:04:14.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.386390 23232 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:14.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.387805 23232 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
02:04:14.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.388329 23232 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d31c363790e4b87aee262915bc24553, State: Initialized, Role: FOLLOWER
02:04:14.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.389441 23232 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:14.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.392794 23357 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:14.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.393831 23357 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.394680 23232 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: configured and running, proceeding with master startup.
02:04:14.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:14.403003 23272 tablet.cc:2406] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:04:14.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.441332 23368 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
02:04:14.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.462622 23157 master_runner.cc:264] No leader master found from master 3d31c363790e4b87aee262915bc24553
02:04:14.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.463258 23157 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:04:14.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:14.538870 23186 tablet.cc:2406] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: Can't schedule compaction. Clean time has not been advanced past its initial value.
02:04:14.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.571444 23329 raft_consensus.cc:493] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:04:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.572388 23329 raft_consensus.cc:515] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.576299 23293 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "149cd51be90048ca9a2b819d788b874a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "55320708ab29420da15b0cb01d4b79f3" is_pre_election: true
02:04:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.577126 23293 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 149cd51be90048ca9a2b819d788b874a in term 0.
02:04:14.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.579165 23207 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "149cd51be90048ca9a2b819d788b874a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d31c363790e4b87aee262915bc24553" is_pre_election: true
02:04:14.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.579898 23207 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 149cd51be90048ca9a2b819d788b874a in term 0.
02:04:14.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.582021 23097 leader_election.cc:304] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 149cd51be90048ca9a2b819d788b874a, 55320708ab29420da15b0cb01d4b79f3; no voters:
02:04:14.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.585778 23329 leader_election.cc:290] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3d31c363790e4b87aee262915bc24553 (127.22.129.189:35809), 55320708ab29420da15b0cb01d4b79f3 (127.22.129.188:40757)
02:04:14.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.586253 23329 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 0 FOLLOWER]: Leader pre-election won for term 1
02:04:14.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.586562 23329 raft_consensus.cc:493] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:04:14.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.586860 23329 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 0 FOLLOWER]: Advancing to term 1
02:04:14.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.603823 23329 raft_consensus.cc:515] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:14.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.605842 23329 leader_election.cc:290] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [CANDIDATE]: Term 1 election: Requested vote from peers 3d31c363790e4b87aee262915bc24553 (127.22.129.189:35809), 55320708ab29420da15b0cb01d4b79f3 (127.22.129.188:40757)
02:04:14.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.606817 23207 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "149cd51be90048ca9a2b819d788b874a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3d31c363790e4b87aee262915bc24553"
02:04:14.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.607352 23207 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 0 FOLLOWER]: Advancing to term 1
02:04:14.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.608906 23293 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "149cd51be90048ca9a2b819d788b874a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "55320708ab29420da15b0cb01d4b79f3"
02:04:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.609441 23293 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [term 0 FOLLOWER]: Advancing to term 1
02:04:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.616050 23207 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 149cd51be90048ca9a2b819d788b874a in term 1.
02:04:14.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.616762 23293 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 149cd51be90048ca9a2b819d788b874a in term 1.
02:04:14.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.617354 23099 leader_election.cc:304] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 149cd51be90048ca9a2b819d788b874a, 3d31c363790e4b87aee262915bc24553; no voters:
02:04:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.621408 23369 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 1 FOLLOWER]: Leader election won for term 1
02:04:14.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.622043 23369 raft_consensus.cc:697] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 1 LEADER]: Becoming Leader. State: Replica: 149cd51be90048ca9a2b819d788b874a, State: Running, Role: LEADER
02:04:14.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.623783 23369 consensus_queue.cc:237] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:14.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.636039 23329 sys_catalog.cc:455] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 149cd51be90048ca9a2b819d788b874a. Latest consensus state: current_term: 1 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:14.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.637531 23329 sys_catalog.cc:458] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: This master's current role is: LEADER
02:04:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.639631 23371 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
02:04:14.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.647157 23371 catalog_manager.cc:1519] Initializing Kudu cluster ID...
02:04:14.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.666154 23207 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Refusing update from remote peer 149cd51be90048ca9a2b819d788b874a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:04:14.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.667891 23369 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:04:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.671890 23293 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [term 1 FOLLOWER]: Refusing update from remote peer 149cd51be90048ca9a2b819d788b874a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
02:04:14.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.674712 23369 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [LEADER]: Connected to new peer: Peer: permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
02:04:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.709560 23335 sys_catalog.cc:455] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 149cd51be90048ca9a2b819d788b874a. Latest consensus state: current_term: 1 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.702150 23357 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 149cd51be90048ca9a2b819d788b874a. Latest consensus state: current_term: 1 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:14.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.710294 23357 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.710742 23335 sys_catalog.cc:458] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.713748 23372 mvcc.cc:204] Tried to move back new op lower bound from 7297333062285987840 to 7297333062150279168. Current Snapshot: MvccSnapshot[applied={T|T < 7297333062285987840}]
02:04:14.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.731038 23357 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:14.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.732040 23374 mvcc.cc:204] Tried to move back new op lower bound from 7297333062285987840 to 7297333062150279168. Current Snapshot: MvccSnapshot[applied={T|T < 7297333062285987840}]
02:04:14.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.732391 23357 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.755448 23369 sys_catalog.cc:455] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:14.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.755491 23335 sys_catalog.cc:455] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:14.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.756148 23369 sys_catalog.cc:458] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: This master's current role is: LEADER
02:04:14.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.756153 23335 sys_catalog.cc:458] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: This master's current role is: FOLLOWER
02:04:14.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.757061 23371 catalog_manager.cc:1382] Generated new cluster ID: 2ff500f327844139b9817693b10d2fc6
02:04:14.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.757300 23371 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
02:04:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.758999 23369 sys_catalog.cc:455] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:14.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.759652 23369 sys_catalog.cc:458] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: This master's current role is: LEADER
02:04:14.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.803558 23371 catalog_manager.cc:1405] Generated new certificate authority record
02:04:14.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.805930 23371 catalog_manager.cc:1539] Loading token signing keys...
02:04:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.836544 23371 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: Generated new TSK 0
02:04:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.837643 23371 catalog_manager.cc:1549] Initializing in-progress tserver states...
02:04:14.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.129.129@KRBTEST.COM; defaulting to no policy
02:04:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.129.129@KRBTEST.COM" created.
02:04:14.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:14.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.129 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:14.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.129 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:14.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.129.129@KRBTEST.COM; defaulting to no policy
02:04:14.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:14.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.129.129@KRBTEST.COM" created.
02:04:14.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:14.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.129 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:14.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.129 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:14.989393 23046 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:14.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/wal
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/data
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/logs
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/data/info.pb
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:14.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.129:0
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.129.129
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.129
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.129
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:14.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:15.128604 23228 debug-util.cc:398] Leaking SignalData structure 0x7b080007da80 after lost signal to thread 23158
02:04:15.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:15.129237 23228 debug-util.cc:398] Leaking SignalData structure 0x7b0800071820 after lost signal to thread 23231
02:04:15.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:15.263466 23353 catalog_manager.cc:1294] Loaded cluster ID: 2ff500f327844139b9817693b10d2fc6
02:04:15.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:15.264261 23353 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: loading cluster ID for follower catalog manager: success
02:04:15.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:15.272271 23353 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: acquiring CA information for follower catalog manager: success
02:04:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:15.302865 23353 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:04:15.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:15.462333 23368 catalog_manager.cc:1294] Loaded cluster ID: 2ff500f327844139b9817693b10d2fc6
02:04:15.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:15.470175 23368 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: loading cluster ID for follower catalog manager: success
02:04:15.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:15.479562 23368 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: acquiring CA information for follower catalog manager: success
02:04:15.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:15.495173 23368 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:04:15.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:15.714073 23385 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:15.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:15.715179 23385 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:15.716754 23385 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:15.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:15.810189 23385 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:15.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:15.812475 23385 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.129.129
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:15.930523 23385 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/data
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/wal
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.129
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.129:0
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:15.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/data/info.pb
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.129
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.129.129
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/logs
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:15.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:04:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:04:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:04:15.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:15.943423 23385 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:15.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:15.945564 23385 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:16.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:16.016117 23393 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:16.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:16.035480 23394 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:18.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:16.040537 23397 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:18.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:16.053126 23385 server_base.cc:1061] running on GCE node
02:04:18.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.163759 23385 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:18.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.179203 23385 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:18.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.181272 23385 hybrid_clock.cc:648] HybridClock initialized: now 1781575458181176 us; error 102 us; skew 500 ppm
02:04:18.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.182844 23385 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:18.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:18 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575458, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:18.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.193941 23385 init.cc:377] Logged in from keytab as kudu/127.22.129.129@KRBTEST.COM (short username kudu)
02:04:18.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.232817 23385 webserver.cc:533] Webserver started at http://127.22.129.129:40619/ using document root <none> and password file <none>
02:04:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.234691 23385 fs_manager.cc:362] Metadata directory not provided
02:04:18.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.235268 23385 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.236171 23385 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.246400 23385 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/data/instance:
02:04:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d2e622126c74f8c80dfd3074850850d"
02:04:18.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:18 on dist-test-slave-nkh1"
02:04:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.248875 23385 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/wal/instance:
02:04:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d2e622126c74f8c80dfd3074850850d"
02:04:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:18 on dist-test-slave-nkh1"
02:04:18.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.270308 23385 fs_manager.cc:696] Time spent creating directory manager: real 0.020s user 0.006s sys 0.012s
02:04:18.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.284783 23410 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.286729 23385 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.005s sys 0.004s
02:04:18.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.287465 23385 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/wal
02:04:18.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0d2e622126c74f8c80dfd3074850850d"
02:04:18.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:18 on dist-test-slave-nkh1"
02:04:18.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.288334 23385 fs_report.cc:389] FS layout report
02:04:18.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:18.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/wal
02:04:18.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/wal
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/data/data
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:18.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.521111 23385 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:18.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.530544 23385 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:18.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.531639 23385 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:18.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.541419 23385 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:18.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.561237 23385 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.561852 23385 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
02:04:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.562387 23385 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.562775 23385 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:04:18.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:18 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575458, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:18.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:18.823422 23092 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:19.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.003254 23385 rpc_server.cc:307] RPC server started. Bound to: 127.22.129.129:42769
02:04:19.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.003378 23523 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.129.129:42769 every 8 connection(s)
02:04:19.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.008208 23385 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/data/info.pb
02:04:19.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.019088 23046 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 23385
02:04:19.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.019656 23046 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-0/wal/instance
02:04:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:19 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575458, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.129@KRBTEST.COM for kudu/127.22.129.189@KRBTEST.COM
02:04:19.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.129.130@KRBTEST.COM; defaulting to no policy
02:04:19.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:19 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575458, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.129@KRBTEST.COM for kudu/127.22.129.188@KRBTEST.COM
02:04:19.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:19.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.129.130@KRBTEST.COM" created.
02:04:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:19 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575458, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.129@KRBTEST.COM for kudu/127.22.129.190@KRBTEST.COM
02:04:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.138803 23527 heartbeater.cc:344] Connected to a master server at 127.22.129.189:35809
02:04:19.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.139670 23527 heartbeater.cc:461] Registering TS with master...
02:04:19.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.141009 23527 heartbeater.cc:507] Master 127.22.129.189:35809 requested a full tablet report, sending...
02:04:19.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.144976 23197 ts_manager.cc:194] Registered new tserver with Master: 0d2e622126c74f8c80dfd3074850850d (127.22.129.129:42769)
02:04:19.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.155061 23524 heartbeater.cc:344] Connected to a master server at 127.22.129.188:40757
02:04:19.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.155421 23524 heartbeater.cc:461] Registering TS with master...
02:04:19.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.156809 23524 heartbeater.cc:507] Master 127.22.129.188:40757 requested a full tablet report, sending...
02:04:19.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.159971 23283 ts_manager.cc:194] Registered new tserver with Master: 0d2e622126c74f8c80dfd3074850850d (127.22.129.129:42769)
02:04:19.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:19.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.130 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:19.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.130 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:19.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:19.162511 23046 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.22.129.130: real 0.132s user 0.000s sys 0.015s
02:04:19.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.165454 23530 heartbeater.cc:344] Connected to a master server at 127.22.129.190:42071
02:04:19.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.165824 23530 heartbeater.cc:461] Registering TS with master...
02:04:19.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.166821 23530 heartbeater.cc:507] Master 127.22.129.190:42071 requested a full tablet report, sending...
02:04:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.170186 23113 ts_manager.cc:194] Registered new tserver with Master: 0d2e622126c74f8c80dfd3074850850d (127.22.129.129:42769)
02:04:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.173000 23113 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.22.129.129@KRBTEST.COM'} at 127.22.129.129:46155
02:04:19.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.129.130@KRBTEST.COM; defaulting to no policy
02:04:19.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:19.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.129.130@KRBTEST.COM" created.
02:04:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.130 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:19.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.130 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.226204 23046 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:19.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/wal
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/data
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/logs
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/data/info.pb
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.130:0
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.129.130
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.130
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.130
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:19.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:19.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:19.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:19.660099 23538 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:19.660748 23538 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:19.661643 23538 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:19.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:19.700505 23538 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:19.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:19.701371 23538 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.129.130
02:04:19.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.753482 23538 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:19.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/data
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/wal
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.130
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.130:0
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/data/info.pb
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.130
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.129.130
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/logs
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:04:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:04:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:04:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.755419 23538 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:19.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:19.756716 23538 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:19.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:19.769006 23545 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:20.178920 23530 heartbeater.cc:499] Master 127.22.129.190:42071 was elected leader, sending a full tablet report...
02:04:21.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:19.774307 23548 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:21.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:19.771507 23546 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:21.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:21.159904 23547 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1385 milliseconds
02:04:21.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.160048 23538 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.161686 23538 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:21.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.164997 23538 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:21.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.166488 23538 hybrid_clock.cc:648] HybridClock initialized: now 1781575461166430 us; error 61 us; skew 500 ppm
02:04:21.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.167757 23538 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:21.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:21 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575461, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.175247 23538 init.cc:377] Logged in from keytab as kudu/127.22.129.130@KRBTEST.COM (short username kudu)
02:04:21.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.181099 23538 webserver.cc:533] Webserver started at http://127.22.129.130:45433/ using document root <none> and password file <none>
02:04:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.182190 23538 fs_manager.cc:362] Metadata directory not provided
02:04:21.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.182438 23538 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:21.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.182967 23538 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.187971 23538 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/data/instance:
02:04:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40a1638e1a1342ea868ee3541a52991c"
02:04:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:21 on dist-test-slave-nkh1"
02:04:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.189285 23538 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/wal/instance:
02:04:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40a1638e1a1342ea868ee3541a52991c"
02:04:21.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:21 on dist-test-slave-nkh1"
02:04:21.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.196758 23538 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
02:04:21.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.202661 23559 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:21.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.203652 23538 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
02:04:21.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.204114 23538 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/wal
02:04:21.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "40a1638e1a1342ea868ee3541a52991c"
02:04:21.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:21 on dist-test-slave-nkh1"
02:04:21.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.204444 23538 fs_report.cc:389] FS layout report
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/wal
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/wal
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/data/data
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:21.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.255988 23538 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:21.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.263151 23538 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:21.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.263667 23538 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:21.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.267112 23538 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:21.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.271237 23538 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:21.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.271500 23538 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:04:21.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.271773 23538 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.271967 23538 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
02:04:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.435773 23538 rpc_server.cc:307] RPC server started. Bound to: 127.22.129.130:39961
02:04:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.435947 23672 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.129.130:39961 every 8 connection(s)
02:04:21.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.438369 23538 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/data/info.pb
02:04:21.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.447197 23046 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 23538
02:04:21.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.447791 23046 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-1/wal/instance
02:04:21.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.22.129.131@KRBTEST.COM; defaulting to no policy
02:04:21.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:21 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575461, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.130@KRBTEST.COM for kudu/127.22.129.188@KRBTEST.COM
02:04:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:21.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.22.129.131@KRBTEST.COM" created.
02:04:21.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:21 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575461, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.130@KRBTEST.COM for kudu/127.22.129.189@KRBTEST.COM
02:04:21.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:21 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575461, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.130@KRBTEST.COM for kudu/127.22.129.190@KRBTEST.COM
02:04:21.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.543156 23675 heartbeater.cc:344] Connected to a master server at 127.22.129.190:42071
02:04:21.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.543675 23675 heartbeater.cc:461] Registering TS with master...
02:04:21.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.545135 23673 heartbeater.cc:344] Connected to a master server at 127.22.129.188:40757
02:04:21.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.545688 23673 heartbeater.cc:461] Registering TS with master...
02:04:21.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.547755 23673 heartbeater.cc:507] Master 127.22.129.188:40757 requested a full tablet report, sending...
02:04:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.549758 23675 heartbeater.cc:507] Master 127.22.129.190:42071 requested a full tablet report, sending...
02:04:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.551900 23283 ts_manager.cc:194] Registered new tserver with Master: 40a1638e1a1342ea868ee3541a52991c (127.22.129.130:39961)
02:04:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.552701 23674 heartbeater.cc:344] Connected to a master server at 127.22.129.189:35809
02:04:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.553064 23674 heartbeater.cc:461] Registering TS with master...
02:04:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.553975 23674 heartbeater.cc:507] Master 127.22.129.189:35809 requested a full tablet report, sending...
02:04:21.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.553822 23113 ts_manager.cc:194] Registered new tserver with Master: 40a1638e1a1342ea868ee3541a52991c (127.22.129.130:39961)
02:04:21.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.555889 23113 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.22.129.130@KRBTEST.COM'} at 127.22.129.130:45229
02:04:21.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.556746 23197 ts_manager.cc:194] Registered new tserver with Master: 40a1638e1a1342ea868ee3541a52991c (127.22.129.130:39961)
02:04:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.131 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:21.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.22.129.131 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:21.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.22.129.131@KRBTEST.COM; defaulting to no policy
02:04:21.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:21.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.22.129.131@KRBTEST.COM" created.
02:04:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
02:04:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.131 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:21.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.22.129.131 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab.
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.605152 23046 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/wal
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/data
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/logs
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/data/info.pb
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:21.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.131:0
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.129.131
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.131
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.131
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:21.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:21.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:21 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575461, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:21.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:21.683679 23176 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:21.946213 23687 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:21.946676 23687 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:21.947180 23687 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:21.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:21.981175 23687 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:21.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:21.982556 23687 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.22.129.131
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:22.027014 23687 tablet_server_runner.cc:78] Tablet server non-default flags:
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/data
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/wal
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.131
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:22.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.131:0
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/data/info.pb
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.131
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.22.129.131
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/logs
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:22.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:22.028410 23687 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:22.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:22.029451 23687 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:22.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:22.041900 23694 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:22.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:22.562438 23675 heartbeater.cc:499] Master 127.22.129.190:42071 was elected leader, sending a full tablet report...
02:04:23.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:23.444308 23693 debug-util.cc:398] Leaking SignalData structure 0x7b08000370e0 after lost signal to thread 23687
02:04:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:23.449357 23693 kernel_stack_watchdog.cc:198] Thread 23687 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
02:04:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
02:04:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
02:04:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
02:04:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
02:04:23.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:22.043506 23695 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:23.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:23.452252 23687 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.408s user 0.001s sys 0.000s
02:04:23.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:23.452598 23687 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.408s user 0.001s sys 0.000s
02:04:23.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.459419 23687 server_base.cc:1061] running on GCE node
02:04:23.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:23.459508 23697 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:23.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.460845 23687 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:23.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.462958 23687 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:23.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.464293 23687 hybrid_clock.cc:648] HybridClock initialized: now 1781575463464252 us; error 44 us; skew 500 ppm
02:04:23.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.465152 23687 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:23.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:23 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575463, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:23.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.471144 23687 init.cc:377] Logged in from keytab as kudu/127.22.129.131@KRBTEST.COM (short username kudu)
02:04:23.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.476378 23687 webserver.cc:533] Webserver started at http://127.22.129.131:46009/ using document root <none> and password file <none>
02:04:23.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.477222 23687 fs_manager.cc:362] Metadata directory not provided
02:04:23.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.477412 23687 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:23.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.477841 23687 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
02:04:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.482223 23687 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/data/instance:
02:04:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3969ada7f114c008c61f5ba90667d5e"
02:04:23.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:23 on dist-test-slave-nkh1"
02:04:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.483263 23687 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/wal/instance:
02:04:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3969ada7f114c008c61f5ba90667d5e"
02:04:23.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:23 on dist-test-slave-nkh1"
02:04:23.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.491899 23687 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.010s sys 0.001s
02:04:23.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.497303 23708 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.498358 23687 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
02:04:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.498644 23687 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/wal
02:04:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3969ada7f114c008c61f5ba90667d5e"
02:04:23.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:23 on dist-test-slave-nkh1"
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.498926 23687 fs_report.cc:389] FS layout report
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/wal
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/wal
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/data/data
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:23.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:23.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.550690 23687 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:23.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.556346 23687 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:23.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.556852 23687 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:23.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.560210 23687 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
02:04:23.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.566428 23687 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
02:04:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.566708 23687 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
02:04:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.566982 23687 ts_tablet_manager.cc:616] Registered 0 tablets
02:04:23.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.567179 23687 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
02:04:23.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.736559 23687 rpc_server.cc:307] RPC server started. Bound to: 127.22.129.131:38245
02:04:23.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.736738 23820 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.129.131:38245 every 8 connection(s)
02:04:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.740190 23687 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/data/info.pb
02:04:23.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.741285 23046 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 23687
02:04:23.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.741801 23046 external_mini_cluster.cc:1448] Reading /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/ts-2/wal/instance
02:04:23.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:23 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575463, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.131@KRBTEST.COM for kudu/127.22.129.189@KRBTEST.COM
02:04:23.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:23 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575463, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.131@KRBTEST.COM for kudu/127.22.129.190@KRBTEST.COM
02:04:23.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:23 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575463, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.131@KRBTEST.COM for kudu/127.22.129.188@KRBTEST.COM
02:04:23.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:23 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575463, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:23.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.801033 23261 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:23.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.810418 23822 heartbeater.cc:344] Connected to a master server at 127.22.129.189:35809
02:04:23.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.810916 23822 heartbeater.cc:461] Registering TS with master...
02:04:23.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.812314 23822 heartbeater.cc:507] Master 127.22.129.189:35809 requested a full tablet report, sending...
02:04:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.815208 23823 heartbeater.cc:344] Connected to a master server at 127.22.129.190:42071
02:04:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.815521 23823 heartbeater.cc:461] Registering TS with master...
02:04:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.816449 23823 heartbeater.cc:507] Master 127.22.129.190:42071 requested a full tablet report, sending...
02:04:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.818849 23197 ts_manager.cc:194] Registered new tserver with Master: e3969ada7f114c008c61f5ba90667d5e (127.22.129.131:38245)
02:04:23.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.821987 23113 ts_manager.cc:194] Registered new tserver with Master: e3969ada7f114c008c61f5ba90667d5e (127.22.129.131:38245)
02:04:23.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.823706 23113 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.22.129.131@KRBTEST.COM'} at 127.22.129.131:36629
02:04:23.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.824334 23821 heartbeater.cc:344] Connected to a master server at 127.22.129.188:40757
02:04:23.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.824690 23821 heartbeater.cc:461] Registering TS with master...
02:04:23.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.825606 23821 heartbeater.cc:507] Master 127.22.129.188:40757 requested a full tablet report, sending...
02:04:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.828600 23283 ts_manager.cc:194] Registered new tserver with Master: e3969ada7f114c008c61f5ba90667d5e (127.22.129.131:38245)
02:04:23.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.842046 23046 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
02:04:23.857 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:23.858 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
02:04:23.866 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
02:04:23.867 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
02:04:23.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.22.129.190"
port: 42071
}
bound_http_address {
host: "127.22.129.190"
port: 42433
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.22.129.189"
port: 35809
}
bound_http_address {
host: "127.22.129.189"
port: 36035
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.22.129.188"
port: 40757
}
bound_http_address {
host: "127.22.129.188"
port: 43773
}
}
}
02:04:23.871 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
02:04:23.873 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.22.129.129"
port: 42769
}
bound_http_address {
host: "127.22.129.129"
port: 40619
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.22.129.130"
port: 39961
}
bound_http_address {
host: "127.22.129.130"
port: 45433
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.22.129.131"
port: 38245
}
bound_http_address {
host: "127.22.129.131"
port: 46009
}
}
}
02:04:23.873 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:04:23.875 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:04:23.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:23 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575463, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:23.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:04:23.916 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:23.919 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc
02:04:23.930 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:04:23.930 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc
02:04:23.932 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:04:23.957 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
02:04:23.958 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
02:04:23.958 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
02:04:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:23 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575463, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:23.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
02:04:23.991 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:23.992 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc
02:04:23.993 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
02:04:23.994 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.129.189:35809
02:04:23.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
02:04:23.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:23.995957 23046 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 23157
02:04:24.033 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:24.033 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.129.190:42071
02:04:24.033 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
02:04:24.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:24.034711 23046 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 23076
02:04:24.068 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:24.069 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.22.129.188:40757
02:04:24.069 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
02:04:24.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:24.069842 23046 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 23242
02:04:24.100 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:24.101 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.129.189:35809
02:04:24.105 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:24.106851 23046 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/logs
02:04:24.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data/info.pb
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.189:35809
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.189
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36035
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.189
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:24.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:24.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.173457 23527 heartbeater.cc:646] Failed to heartbeat to 127.22.129.189:35809 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.22.129.189:35809: connect: Connection refused (error 111)
02:04:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.425595 23836 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.426144 23836 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:24.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.426739 23836 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:24.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.461455 23836 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:24.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.461705 23836 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:24.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.461978 23836 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:24.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.462178 23836 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:24.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.462375 23836 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:24.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.462564 23836 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:24.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.462719 23836 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37909/jwks.json
02:04:24.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:24.503283 23836 master_runner.cc:387] Master server non-default flags:
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.189
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.189:35809
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data/info.pb
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.189
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36035
02:04:24.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/logs
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:24.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:24.505102 23836 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:24.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:24.506536 23836 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:24.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.517558 23843 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:24.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.588639 23673 heartbeater.cc:646] Failed to heartbeat to 127.22.129.188:40757 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:24.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:24.829111 23823 heartbeater.cc:499] Master 127.22.129.190:42071 was elected leader, sending a full tablet report...
02:04:24.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.846676 23822 heartbeater.cc:646] Failed to heartbeat to 127.22.129.189:35809 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.22.129.189:35809: connect: Connection refused (error 111)
02:04:26.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:25.929086 23842 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 23836
02:04:26.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:24.518036 23844 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:26.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.263765 23836 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.746s user 0.579s sys 1.081s
02:04:26.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.264320 23836 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.747s user 0.579s sys 1.082s
02:04:26.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.266085 23846 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:26.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.270279 23845 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1751 milliseconds
02:04:26.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.270315 23836 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:26.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.271639 23836 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:26.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.274027 23836 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:26.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.275413 23836 hybrid_clock.cc:648] HybridClock initialized: now 1781575466275352 us; error 72 us; skew 500 ppm
02:04:26.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.276432 23836 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:26.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:26 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575466, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:26.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.283130 23836 init.cc:377] Logged in from keytab as kudu/127.22.129.189@KRBTEST.COM (short username kudu)
02:04:26.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.289569 23836 webserver.cc:533] Webserver started at http://127.22.129.189:36035/ using document root <none> and password file <none>
02:04:26.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.290566 23836 fs_manager.cc:362] Metadata directory not provided
02:04:26.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.290822 23836 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:26.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.299080 23836 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.002s sys 0.004s
02:04:26.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.304087 23873 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:26.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.305274 23836 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.003s
02:04:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.305721 23836 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal
02:04:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3d31c363790e4b87aee262915bc24553"
02:04:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:11 on dist-test-slave-nkh1"
02:04:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.308070 23836 fs_report.cc:389] FS layout report
02:04:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal
02:04:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/wal
02:04:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data/data
02:04:26.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:26.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:26.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.367400 23836 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:26.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.376556 23836 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:26.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.377089 23836 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.454679 23836 rpc_server.cc:307] RPC server started. Bound to: 127.22.129.189:35809
02:04:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.454739 23924 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.129.189:35809 every 8 connection(s)
02:04:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.457811 23836 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-1/data/info.pb
02:04:26.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.458762 23046 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 23836
02:04:26.459 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:26.459 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.129.190:42071
02:04:26.460 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
02:04:26.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.462157 23046 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:26.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:26.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal
02:04:26.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data
02:04:26.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/logs
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data/info.pb
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.190:42071
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.190
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42433
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.190
02:04:26.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:26.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:26.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.471580 23925 sys_catalog.cc:263] Verifying existing consensus state
02:04:26.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.477082 23925 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: Bootstrap starting.
02:04:26.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.516570 23925 log.cc:826] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: Log is configured to *not* fsync() on all Append() calls
02:04:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.532858 23925 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:26.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.533756 23925 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: Bootstrap complete.
02:04:26.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.558938 23925 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:26.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.562769 23925 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3d31c363790e4b87aee262915bc24553, State: Initialized, Role: FOLLOWER
02:04:26.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.563891 23925 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:26.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.574452 23929 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:26.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.575542 23929 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: This master's current role is: FOLLOWER
02:04:26.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.576308 23925 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: configured and running, proceeding with master startup.
02:04:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.620687 23940 catalog_manager.cc:1294] Loaded cluster ID: 2ff500f327844139b9817693b10d2fc6
02:04:26.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.621011 23940 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: loading cluster ID for follower catalog manager: success
02:04:26.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.636926 23940 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: acquiring CA information for follower catalog manager: success
02:04:26.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.607242 23836 master_runner.cc:187] Error getting master registration for 127.22.129.190:42071: Network error: Client connection negotiation failed: client connection to 127.22.129.190:42071: connect: Connection refused (error 111),
02:04:26.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.638130 23836 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:04:26.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.643270 23940 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:04:26.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.677762 23674 heartbeater.cc:344] Connected to a master server at 127.22.129.189:35809
02:04:26.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.682736 23890 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "40a1638e1a1342ea868ee3541a52991c" instance_seqno: 1781575461399945) as {username='kudu', principal='kudu/127.22.129.130@KRBTEST.COM'} at 127.22.129.130:56819; Asking this server to re-register.
02:04:26.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.684357 23674 heartbeater.cc:461] Registering TS with master...
02:04:26.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.687287 23890 ts_manager.cc:194] Registered new tserver with Master: 40a1638e1a1342ea868ee3541a52991c (127.22.129.130:39961)
02:04:26.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.859025 23927 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:26.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.860222 23927 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:26.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.861057 23927 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:26.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.885448 23822 heartbeater.cc:344] Connected to a master server at 127.22.129.189:35809
02:04:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.888613 23890 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e3969ada7f114c008c61f5ba90667d5e" instance_seqno: 1781575463697689) as {username='kudu', principal='kudu/127.22.129.131@KRBTEST.COM'} at 127.22.129.131:37629; Asking this server to re-register.
02:04:26.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.890466 23822 heartbeater.cc:461] Registering TS with master...
02:04:26.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.893687 23890 ts_manager.cc:194] Registered new tserver with Master: e3969ada7f114c008c61f5ba90667d5e (127.22.129.131:38245)
02:04:26.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.917222 23927 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:26.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.917563 23927 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:26.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.918081 23927 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:26.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.918468 23927 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:26.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.918852 23927 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:26.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.919209 23927 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:26.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:26.919447 23927 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37909/jwks.json
02:04:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.989877 23927 master_runner.cc:387] Master server non-default flags:
02:04:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data
02:04:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal
02:04:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:26.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.190
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.190:42071
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data/info.pb
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.190
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42433
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/logs
02:04:26.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:26.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:26.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:26.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:26.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:26.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:04:26.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:26.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:04:26.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:04:26.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:26.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.991557 23927 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:26.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:26.993038 23927 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:27.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:27.017558 23953 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:27.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:27.255187 23527 heartbeater.cc:344] Connected to a master server at 127.22.129.189:35809
02:04:27.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:27.258559 23890 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0d2e622126c74f8c80dfd3074850850d" instance_seqno: 1781575458950191) as {username='kudu', principal='kudu/127.22.129.129@KRBTEST.COM'} at 127.22.129.129:44411; Asking this server to re-register.
02:04:27.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:27.260150 23527 heartbeater.cc:461] Registering TS with master...
02:04:27.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:27.263706 23890 ts_manager.cc:194] Registered new tserver with Master: 0d2e622126c74f8c80dfd3074850850d (127.22.129.129:42769)
02:04:28.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:28 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575468, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:28.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:28.220588 23406 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:28.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:28.380303 23968 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:04:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:28.386982 23968 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:28.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:28.402783 23877 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.129.190:42071: connect: Connection refused (error 111)
02:04:28.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:28.408401 23968 leader_election.cc:290] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 149cd51be90048ca9a2b819d788b874a (127.22.129.190:42071), 55320708ab29420da15b0cb01d4b79f3 (127.22.129.188:40757)
02:04:28.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:28.418215 23874 leader_election.cc:336] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 55320708ab29420da15b0cb01d4b79f3 (127.22.129.188:40757): Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:28.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:28.425711 23877 leader_election.cc:336] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 149cd51be90048ca9a2b819d788b874a (127.22.129.190:42071): Network error: Client connection negotiation failed: client connection to 127.22.129.190:42071: connect: Connection refused (error 111)
02:04:28.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:28.426662 23877 leader_election.cc:304] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d31c363790e4b87aee262915bc24553; no voters: 149cd51be90048ca9a2b819d788b874a, 55320708ab29420da15b0cb01d4b79f3
02:04:28.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:28.427996 23968 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:04:29.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:29.092073 22460 debug-util.cc:398] Leaking SignalData structure 0x7b080008bdc0 after lost signal to thread 22395
02:04:29.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:29.092900 22460 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae8a0 after lost signal to thread 22463
02:04:30.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:28.420646 23952 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 23927
02:04:30.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:27.021073 23954 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:30.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:30.266076 23955 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3244 milliseconds
02:04:30.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:30.265141 23927 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.247s user 0.947s sys 1.797s
02:04:30.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:30.267359 23927 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.249s user 0.947s sys 1.797s
02:04:30.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:30.267426 23956 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:30.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.267755 23927 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:30.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.273032 23927 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:30.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.276197 23927 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:30.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.277631 23927 hybrid_clock.cc:648] HybridClock initialized: now 1781575470277589 us; error 45 us; skew 500 ppm
02:04:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.278734 23927 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:30 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575470, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:30.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.286533 23985 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:04:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.288203 23927 init.cc:377] Logged in from keytab as kudu/127.22.129.190@KRBTEST.COM (short username kudu)
02:04:30.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.287819 23985 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:30.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.313297 23927 webserver.cc:533] Webserver started at http://127.22.129.190:42433/ using document root <none> and password file <none>
02:04:30.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.314606 23927 fs_manager.cc:362] Metadata directory not provided
02:04:30.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.314922 23927 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:30.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.334234 23927 fs_manager.cc:714] Time spent opening directory manager: real 0.014s user 0.005s sys 0.006s
02:04:30.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.338768 23985 leader_election.cc:290] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 149cd51be90048ca9a2b819d788b874a (127.22.129.190:42071), 55320708ab29420da15b0cb01d4b79f3 (127.22.129.188:40757)
02:04:30.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.346796 23995 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:30.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.348096 23927 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.001s sys 0.008s
02:04:30.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.348466 23927 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "149cd51be90048ca9a2b819d788b874a"
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:08 on dist-test-slave-nkh1"
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.351182 23927 fs_report.cc:389] FS layout report
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/wal
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data/data
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:30.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:30.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:30.355822 23877 leader_election.cc:336] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 149cd51be90048ca9a2b819d788b874a (127.22.129.190:42071): Network error: Client connection negotiation failed: client connection to 127.22.129.190:42071: connect: Connection refused (error 111)
02:04:30.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:30.360751 23874 leader_election.cc:336] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 55320708ab29420da15b0cb01d4b79f3 (127.22.129.188:40757): Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:30.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.361385 23874 leader_election.cc:304] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 3d31c363790e4b87aee262915bc24553; no voters: 149cd51be90048ca9a2b819d788b874a, 55320708ab29420da15b0cb01d4b79f3
02:04:30.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.362296 23985 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
02:04:30.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:30.428251 23817 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae040 after lost signal to thread 23688
02:04:30.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:30.429438 23817 debug-util.cc:398] Leaking SignalData structure 0x7b08000b98a0 after lost signal to thread 23820
02:04:30.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.429560 23927 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:30.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.437057 23927 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:30.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.437589 23927 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.590600 23927 rpc_server.cc:307] RPC server started. Bound to: 127.22.129.190:42071
02:04:30.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.590767 24046 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.129.190:42071 every 8 connection(s)
02:04:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.593730 23927 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-0/data/info.pb
02:04:30.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.594473 23046 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 23927
02:04:30.595 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:30.596 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.22.129.188:40757
02:04:30.596 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.598809 23046 external_mini_cluster.cc:1372] Running /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-task4yviL0/build/tsan/bin/kudu
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/logs
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data/info.pb
02:04:30.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:30.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
02:04:30.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.188:40757
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.188
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43773
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.188
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:30.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
02:04:30.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
02:04:30.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.637274 24047 sys_catalog.cc:263] Verifying existing consensus state
02:04:30.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.647297 24047 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: Bootstrap starting.
02:04:30.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.702841 24047 log.cc:826] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: Log is configured to *not* fsync() on all Append() calls
02:04:30.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.783761 24047 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:30.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.785954 24047 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: Bootstrap complete.
02:04:30.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.822510 24047 raft_consensus.cc:359] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:30.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.826347 24047 raft_consensus.cc:740] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 149cd51be90048ca9a2b819d788b874a, State: Initialized, Role: FOLLOWER
02:04:30.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.827580 24047 consensus_queue.cc:260] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:30.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.837967 23675 heartbeater.cc:344] Connected to a master server at 127.22.129.190:42071
02:04:30.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.859344 24054 sys_catalog.cc:455] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:30.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.860205 24054 sys_catalog.cc:458] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: This master's current role is: FOLLOWER
02:04:30.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:30.861930 24047 sys_catalog.cc:565] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: configured and running, proceeding with master startup.
02:04:31.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.006464 23823 heartbeater.cc:344] Connected to a master server at 127.22.129.190:42071
02:04:31.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.007182 23823 heartbeater.cc:499] Master 127.22.129.190:42071 was elected leader, sending a full tablet report...
02:04:31.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.016039 24012 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "e3969ada7f114c008c61f5ba90667d5e" instance_seqno: 1781575463697689) as {username='kudu', principal='kudu/127.22.129.131@KRBTEST.COM'} at 127.22.129.131:60891; Asking this server to re-register.
02:04:31.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.018854 23823 heartbeater.cc:461] Registering TS with master...
02:04:31.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.022890 24012 ts_manager.cc:194] Registered new tserver with Master: e3969ada7f114c008c61f5ba90667d5e (127.22.129.131:38245)
02:04:31.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.051838 24067 catalog_manager.cc:1294] Loaded cluster ID: 2ff500f327844139b9817693b10d2fc6
02:04:31.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.052160 24067 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: loading cluster ID for follower catalog manager: success
02:04:31.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:31 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575470, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.190@KRBTEST.COM for kudu/127.22.129.190@KRBTEST.COM
02:04:31.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.112057 24067 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: acquiring CA information for follower catalog manager: success
02:04:31.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.132941 24067 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:04:31.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:31 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575471, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:31.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.180877 23555 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:31.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.171656 23927 master_runner.cc:187] Error getting master registration for 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111),
02:04:31.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.245365 23927 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:04:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.378752 23530 heartbeater.cc:344] Connected to a master server at 127.22.129.190:42071
02:04:31.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.382525 24012 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0d2e622126c74f8c80dfd3074850850d" instance_seqno: 1781575458950191) as {username='kudu', principal='kudu/127.22.129.129@KRBTEST.COM'} at 127.22.129.129:55465; Asking this server to re-register.
02:04:31.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.384189 23530 heartbeater.cc:461] Registering TS with master...
02:04:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.387432 24012 ts_manager.cc:194] Registered new tserver with Master: 0d2e622126c74f8c80dfd3074850850d (127.22.129.129:42769)
02:04:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.420836 24048 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
02:04:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.421638 24048 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
02:04:31.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.422745 24048 flags.cc:432] Enabled unsafe flag: --never_fsync=true
02:04:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.486768 24048 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
02:04:31.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.487142 24048 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
02:04:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.487630 24048 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
02:04:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.488001 24048 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
02:04:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.488340 24048 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
02:04:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.488673 24048 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
02:04:31.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.488915 24048 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37909/jwks.json
02:04:31.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.577184 24048 master_runner.cc:387] Master server non-default flags:
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.22.129.148:43463
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.22.129.190:42071,127.22.129.189:35809,127.22.129.188:40757
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/krb5kdc/kudu.keytab
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.22.129.188
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.22.129.188:40757
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
02:04:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37909/jwks.json
02:04:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
02:04:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
02:04:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
02:04:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
02:04:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
02:04:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data/info.pb
02:04:31.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.22.129.188
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43773
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/logs
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3396d19fb3cae30ffddb1de1e8c4632645aea710
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 16 Jun 2026 01:43:22 UTC on e7f111948823
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12415
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
02:04:31.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.579010 24048 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
02:04:31.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.580374 24048 file_cache.cc:492] Constructed file cache file cache with capacity 419430
02:04:31.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.633332 24079 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:31.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.739106 23836 master_runner.cc:187] Error getting master registration for 127.22.129.188:40757: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111),
02:04:31.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.739564 23836 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
02:04:31.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.846313 24012 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "40a1638e1a1342ea868ee3541a52991c" instance_seqno: 1781575461399945) as {username='kudu', principal='kudu/127.22.129.130@KRBTEST.COM'} at 127.22.129.130:58215; Asking this server to re-register.
02:04:31.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.850126 23675 heartbeater.cc:461] Registering TS with master...
02:04:31.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:31.854945 24012 ts_manager.cc:194] Registered new tserver with Master: 40a1638e1a1342ea868ee3541a52991c (127.22.129.130:39961)
02:04:32.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.324733 24091 raft_consensus.cc:493] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
02:04:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.325503 24091 raft_consensus.cc:515] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:32.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.356792 23900 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "149cd51be90048ca9a2b819d788b874a" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "3d31c363790e4b87aee262915bc24553" is_pre_election: true
02:04:32.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.357633 23900 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 149cd51be90048ca9a2b819d788b874a in term 1.
02:04:32.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.359181 23998 leader_election.cc:304] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 149cd51be90048ca9a2b819d788b874a, 3d31c363790e4b87aee262915bc24553; no voters:
02:04:32.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.378247 24091 leader_election.cc:290] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3d31c363790e4b87aee262915bc24553 (127.22.129.189:35809), 55320708ab29420da15b0cb01d4b79f3 (127.22.129.188:40757)
02:04:32.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.379021 24091 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 1 FOLLOWER]: Leader pre-election won for term 2
02:04:32.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.379465 24091 raft_consensus.cc:493] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
02:04:32.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.390410 24091 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 1 FOLLOWER]: Advancing to term 2
02:04:32.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:32.404572 23996 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:32.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:32.426811 23996 leader_election.cc:336] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 55320708ab29420da15b0cb01d4b79f3 (127.22.129.188:40757): Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:32.429286 23998 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.22.129.189:35809, user_credentials={real_user=slave}} blocked reactor thread for 70283us
02:04:32.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.482465 24091 raft_consensus.cc:515] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:32.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.505292 23900 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "149cd51be90048ca9a2b819d788b874a" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "3d31c363790e4b87aee262915bc24553"
02:04:32.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.506057 23900 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 1 FOLLOWER]: Advancing to term 2
02:04:32.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:32.531606 23996 leader_election.cc:336] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 55320708ab29420da15b0cb01d4b79f3 (127.22.129.188:40757): Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111)
02:04:32.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.534675 24091 leader_election.cc:290] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [CANDIDATE]: Term 2 election: Requested vote from peers 3d31c363790e4b87aee262915bc24553 (127.22.129.189:35809), 55320708ab29420da15b0cb01d4b79f3 (127.22.129.188:40757)
02:04:32.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.555917 23900 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 149cd51be90048ca9a2b819d788b874a in term 2.
02:04:32.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.558171 23998 leader_election.cc:304] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 149cd51be90048ca9a2b819d788b874a, 3d31c363790e4b87aee262915bc24553; no voters: 55320708ab29420da15b0cb01d4b79f3
02:04:32.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.559837 24091 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 2 FOLLOWER]: Leader election won for term 2
02:04:32.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.560953 24091 raft_consensus.cc:697] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [term 2 LEADER]: Becoming Leader. State: Replica: 149cd51be90048ca9a2b819d788b874a, State: Running, Role: LEADER
02:04:32.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.562351 24091 consensus_queue.cc:237] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:32.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.579288 24092 sys_catalog.cc:455] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 149cd51be90048ca9a2b819d788b874a. Latest consensus state: current_term: 2 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:32.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.580976 24092 sys_catalog.cc:458] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: This master's current role is: LEADER
02:04:32.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.592628 24097 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
02:04:32.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.605278 24097 catalog_manager.cc:1519] Initializing Kudu cluster ID...
02:04:32.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.611228 24097 catalog_manager.cc:1294] Loaded cluster ID: 2ff500f327844139b9817693b10d2fc6
02:04:32.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.611778 24097 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
02:04:32.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.618597 24097 catalog_manager.cc:1539] Loading token signing keys...
02:04:32.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.624219 24097 catalog_manager.cc:6080] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a: Loaded TSK: 0
02:04:32.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:32.627070 24097 catalog_manager.cc:1549] Initializing in-progress tserver states...
02:04:32.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:32.953040 23996 consensus_peers.cc:597] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a -> Peer 55320708ab29420da15b0cb01d4b79f3 (127.22.129.188:40757): Couldn't send request to peer 55320708ab29420da15b0cb01d4b79f3. Status: Network error: Client connection negotiation failed: client connection to 127.22.129.188:40757: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
02:04:33.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.026432 23900 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [term 2 FOLLOWER]: Refusing update from remote peer 149cd51be90048ca9a2b819d788b874a: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
02:04:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.030190 24092 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [LEADER]: Connected to new peer: Peer: permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
02:04:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.074855 24100 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 149cd51be90048ca9a2b819d788b874a. Latest consensus state: current_term: 2 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:33.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.089847 24100 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: This master's current role is: FOLLOWER
02:04:33.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.100495 24100 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:33.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.101661 24100 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3d31c363790e4b87aee262915bc24553 [sys.catalog]: This master's current role is: FOLLOWER
02:04:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.110159 24091 sys_catalog.cc:455] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:33.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.110865 24091 sys_catalog.cc:458] T 00000000000000000000000000000000 P 149cd51be90048ca9a2b819d788b874a [sys.catalog]: This master's current role is: LEADER
02:04:33.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:33 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575473, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:33.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.478488 23704 init.cc:295] Successfully reacquired a new kerberos TGT
02:04:33.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.861471 23675 heartbeater.cc:499] Master 127.22.129.190:42071 was elected leader, sending a full tablet report...
02:04:33.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.646571 24080 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:33.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:31.664064 24084 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
02:04:33.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:33.948203 24081 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2299 milliseconds
02:04:33.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.948374 24048 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
02:04:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.949975 24048 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
02:04:33.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.960429 24048 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
02:04:33.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.964089 24048 hybrid_clock.cc:648] HybridClock initialized: now 1781575473964039 us; error 53 us; skew 500 ppm
02:04:33.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.965291 24048 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
02:04:33.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:33 dist-test-slave-nkh1 krb5kdc[23058](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575473, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
02:04:33.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.974073 24048 init.cc:377] Logged in from keytab as kudu/127.22.129.188@KRBTEST.COM (short username kudu)
02:04:33.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.980544 24048 webserver.cc:533] Webserver started at http://127.22.129.188:43773/ using document root <none> and password file <none>
02:04:33.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.981818 24048 fs_manager.cc:362] Metadata directory not provided
02:04:33.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.982132 24048 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
02:04:33.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:33.994328 24048 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.008s sys 0.000s
02:04:34.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.019475 24117 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
02:04:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.021126 24048 fs_manager.cc:730] Time spent opening block manager: real 0.024s user 0.005s sys 0.016s
02:04:34.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.021591 24048 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data,/tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "55320708ab29420da15b0cb01d4b79f3"
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-16 02:04:13 on dist-test-slave-nkh1"
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.024562 24048 fs_report.cc:389] FS layout report
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/wal
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data/data
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
02:04:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
02:04:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.036273 23823 heartbeater.cc:499] Master 127.22.129.190:42071 was elected leader, sending a full tablet report...
02:04:34.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.106666 24048 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
02:04:34.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.120237 24048 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
02:04:34.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.120880 24048 kserver.cc:163] Server-wide thread pool size limit: 3276
02:04:34.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.241380 24048 rpc_server.cc:307] RPC server started. Bound to: 127.22.129.188:40757
02:04:34.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.242036 24169 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.22.129.188:40757 every 8 connection(s)
02:04:34.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.245901 24048 server_base.cc:1193] Dumped server information to /tmp/dist-test-task4yviL0/test-tmp/mini-kudu-cluster7709527448955017773/master-2/data/info.pb
02:04:34.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.250733 23046 external_mini_cluster.cc:1434] Started /tmp/dist-test-task4yviL0/build/tsan/bin/kudu as pid 24048
02:04:34.251 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
02:04:34.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.265231 24170 sys_catalog.cc:263] Verifying existing consensus state
02:04:34.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.274920 24170 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: Bootstrap starting.
02:04:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:34.287441 22460 debug-util.cc:398] Leaking SignalData structure 0x7b080003d260 after lost signal to thread 22395
02:04:34.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:34.288349 22460 debug-util.cc:398] Leaking SignalData structure 0x7b08000b1160 after lost signal to thread 22463
02:04:34.338 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.129.189:35809(127.22.129.189:35809)] connecting to peer
02:04:34.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.403928 24170 log.cc:826] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: Log is configured to *not* fsync() on all Append() calls
02:04:34.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.436197 24170 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
02:04:34.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.437379 24170 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: Bootstrap complete.
02:04:34.456 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.129.190:42071(127.22.129.190:42071)] connecting to peer
02:04:34.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:34 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575468, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.129@KRBTEST.COM for kudu/127.22.129.188@KRBTEST.COM
02:04:34.478 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.22.129.188:40757(127.22.129.188:40757)] connecting to peer
02:04:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.484503 23524 heartbeater.cc:344] Connected to a master server at 127.22.129.188:40757
02:04:34.496 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.22.129.188:40757(127.22.129.188:40757)] Successfully connected to peer
02:04:34.496 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.22.129.189:35809(127.22.129.189:35809)] Successfully connected to peer
02:04:34.500 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.22.129.190:42071(127.22.129.190:42071)] Successfully connected to peer
02:04:34.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.506812 23530 heartbeater.cc:499] Master 127.22.129.190:42071 was elected leader, sending a full tablet report...
02:04:34.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.537093 24170 raft_consensus.cc:359] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:34.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.540220 24170 raft_consensus.cc:740] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 55320708ab29420da15b0cb01d4b79f3, State: Initialized, Role: FOLLOWER
02:04:34.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.542316 24170 consensus_queue.cc:260] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } }
02:04:34.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.550624 24170 sys_catalog.cc:565] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: configured and running, proceeding with master startup.
02:04:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.564440 24182 sys_catalog.cc:455] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:34.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.565608 24182 sys_catalog.cc:458] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: This master's current role is: FOLLOWER
02:04:34.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:34 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575470, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.190@KRBTEST.COM for kudu/127.22.129.188@KRBTEST.COM
02:04:34.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.657030 24145 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [term 1 FOLLOWER]: Advancing to term 2
02:04:34.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.664628 24195 catalog_manager.cc:1294] Loaded cluster ID: 2ff500f327844139b9817693b10d2fc6
02:04:34.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.667673 24195 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: loading cluster ID for follower catalog manager: success
02:04:34.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 16 02:04:34 dist-test-slave-nkh1 krb5kdc[23058](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781575473, etypes {rep=17 tkt=17 ses=17}, kudu/127.22.129.188@KRBTEST.COM for kudu/127.22.129.190@KRBTEST.COM
02:04:34.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.688421 24195 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: acquiring CA information for follower catalog manager: success
02:04:34.693 [WARN - kudu-nio-2] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
02:04:34.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.708015 24182 sys_catalog.cc:455] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 149cd51be90048ca9a2b819d788b874a. Latest consensus state: current_term: 2 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:34.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.709151 24182 sys_catalog.cc:458] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: This master's current role is: FOLLOWER
02:04:34.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.712546 24201 sys_catalog.cc:455] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "149cd51be90048ca9a2b819d788b874a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "149cd51be90048ca9a2b819d788b874a" member_type: VOTER last_known_addr { host: "127.22.129.190" port: 42071 } } peers { permanent_uuid: "3d31c363790e4b87aee262915bc24553" member_type: VOTER last_known_addr { host: "127.22.129.189" port: 35809 } } peers { permanent_uuid: "55320708ab29420da15b0cb01d4b79f3" member_type: VOTER last_known_addr { host: "127.22.129.188" port: 40757 } } }
02:04:34.713 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.22.129.190:42071(127.22.129.190:42071)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:04:34.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.714432 24201 sys_catalog.cc:458] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3 [sys.catalog]: This master's current role is: FOLLOWER
02:04:34.715 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.22.129.189:35809(127.22.129.189:35809)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:04:34.716 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.22.129.190:42071: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:04:34.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.715761 24195 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 55320708ab29420da15b0cb01d4b79f3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
02:04:34.716 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.22.129.189:35809: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:04:34.719 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.22.129.189:35809(127.22.129.189:35809)] handling channelInactive
02:04:34.721 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.22.129.190:42071(127.22.129.190:42071)] handling channelInactive
02:04:34.721 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.22.129.188:40757(127.22.129.188:40757)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:04:34.722 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.22.129.188:40757: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
02:04:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:34.720458 24180 negotiation.cc:336] Failed RPC negotiation. Trace:
02:04:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.490420 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:44358 (local address 127.22.129.189:35809)
02:04:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.505538 (+ 15118us) server_negotiation.cc:207] Beginning negotiation
02:04:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.505589 (+ 51us) server_negotiation.cc:400] Waiting for connection header
02:04:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.577629 (+ 72040us) server_negotiation.cc:408] Connection header received
02:04:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.650458 (+ 72829us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:04:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.650498 (+ 40us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:04:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.650858 (+ 360us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:04:34.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.651111 (+ 253us) server_negotiation.cc:227] Negotiated authn=SASL
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.718870 (+ 67759us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:44358: BlockingRecv error: recv got EOF from 127.0.0.1:44358 (error 108)
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":14017,"thread_start_us":13474,"threads_started":1}
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:34.722349 24181 negotiation.cc:336] Failed RPC negotiation. Trace:
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.495606 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:48180 (local address 127.22.129.190:42071)
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.506741 (+ 11135us) server_negotiation.cc:207] Beginning negotiation
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.506769 (+ 28us) server_negotiation.cc:400] Waiting for connection header
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.577236 (+ 70467us) server_negotiation.cc:408] Connection header received
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.658686 (+ 81450us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.658715 (+ 29us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.658938 (+ 223us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.659103 (+ 165us) server_negotiation.cc:227] Negotiated authn=SASL
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.720932 (+ 61829us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:48180: BlockingRecv error: recv got EOF from 127.0.0.1:48180 (error 108)
02:04:34.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":10572,"thread_start_us":11096,"threads_started":1}
02:04:34.741 [DEBUG - kudu-nio-2] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=485), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.129.189:35809, [191ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.129.190:42071, [212ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.129.188:40757, [450ms] Sub RPC ConnectToMaster: received response from server master-127.22.129.190:42071: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [451ms] Sub RPC ConnectToMaster: received response from server master-127.22.129.189:35809: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [457ms] Sub RPC ConnectToMaster: received response from server master-127.22.129.188:40757: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.129.189:35809,127.22.129.190:42071,127.22.129.188:40757). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.129.189:35809,127.22.129.190:42071,127.22.129.188:40757). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
02:04:34.757 [DEBUG - kudu-nio-2] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=502), Traces: [0ms] refreshing cache from master, [37ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.129.189:35809, [191ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.129.190:42071, [212ms] Sub RPC ConnectToMaster: sending RPC to server master-127.22.129.188:40757, [450ms] Sub RPC ConnectToMaster: received response from server master-127.22.129.190:42071: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [451ms] Sub RPC ConnectToMaster: received response from server master-127.22.129.189:35809: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [457ms] Sub RPC ConnectToMaster: received response from server master-127.22.129.188:40757: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
02:04:34.760 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.129.190:42071(127.22.129.190:42071)] disconnecting while in state TERMINATED
02:04:34.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:34.760341 24176 negotiation.cc:336] Failed RPC negotiation. Trace:
02:04:34.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.492183 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:58082 (local address 127.22.129.188:40757)
02:04:34.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.493441 (+ 1258us) server_negotiation.cc:207] Beginning negotiation
02:04:34.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.493471 (+ 30us) server_negotiation.cc:400] Waiting for connection header
02:04:34.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.580028 (+ 86557us) server_negotiation.cc:408] Connection header received
02:04:34.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.651057 (+ 71029us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
02:04:34.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.651089 (+ 32us) server_negotiation.cc:462] Received NEGOTIATE request from client
02:04:34.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.651316 (+ 227us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
02:04:34.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.651498 (+ 182us) server_negotiation.cc:227] Negotiated authn=SASL
02:04:34.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.758278 (+106780us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:58082: BlockingRecv error: recv got EOF from 127.0.0.1:58082 (error 108)
02:04:34.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":617}
02:04:34.762 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.129.188:40757(127.22.129.188:40757)] disconnecting while in state TERMINATED
02:04:34.762 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.22.129.189:35809(127.22.129.189:35809)] disconnecting while in state TERMINATED
02:04:34.763 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.22.129.188:40757(127.22.129.188:40757)] handling channelInactive
02:04:34.764 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:04:34.770 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:04:34.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.786936 23046 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 23385
02:04:34.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.839363 23046 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 23538
02:04:34.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260616 02:04:34.868844 24187 negotiation.cc:336] Failed RPC negotiation. Trace:
02:04:34.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.827779 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.22.129.130:49693 (local address 127.22.129.188:40757)
02:04:34.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.828674 (+ 895us) server_negotiation.cc:207] Beginning negotiation
02:04:34.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.828705 (+ 31us) server_negotiation.cc:400] Waiting for connection header
02:04:34.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0616 02:04:34.867836 (+ 39131us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.22.129.130:49693: BlockingRecv error: recv got EOF from 127.22.129.130:49693 (error 108)
02:04:34.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":216}
02:04:34.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.872439 23046 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 23687
02:04:34.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.925227 23046 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 23927
02:04:34.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.971289 23046 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 23836
02:04:34.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:34.996471 23046 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 24048
02:04:36.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T02:04:36Z chronyd exiting
02:04:37.268 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
02:04:37.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:37.271219 22309 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 22597
02:04:37.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:37.300823 22309 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 22751
02:04:37.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:37.334743 22309 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 22898
02:04:37.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:37.360035 22309 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 22321
02:04:37.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:37.388171 22309 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 22391
02:04:37.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260616 02:04:37.419648 22309 external_mini_cluster.cc:1664] Killing /tmp/dist-test-task4yviL0/build/tsan/bin/kudu with pid 22466
02:04:38.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-16T02:04:38Z chronyd exiting
02:04:39.677 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.129.189:35809,127.22.129.190:42071,127.22.129.188:40757). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.129.189:35809,127.22.129.190:42071,127.22.129.188:40757). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
02:04:40.899 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
02:04:40.900 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.129.189:35809,127.22.129.190:42071,127.22.129.188:40757). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.22.129.189:35809,127.22.129.190:42071,127.22.129.188:40757). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more