08:03:37.309 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
08:03:37.322 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskexLxKm/test-tmp
08:03:37.328 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:03:37.331 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskexLxKm/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:03:37.539 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
08:03:37.723 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:37.724 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:03:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:37.728315 1461 mini_oidc.cc:150] Starting JWKS server
08:03:37.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:37.731870 1461 webserver.cc:533] Webserver started at http://127.0.0.1:44381/ using document root <none> and password file <none>
08:03:37.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:37.732342 1461 mini_oidc.cc:186] Starting OIDC Discovery server
08:03:37.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:37.734579 1461 webserver.cc:533] Webserver started at http://127.0.0.1:40647/ using document root <none> and password file <none>
08:03:37.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T08:03:37Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:03:37.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T08:03:37Z Disabled control of system clock
08:03:37.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:37.796325 1461 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:37.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:37.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/wal
08:03:37.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/data
08:03:37.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:37.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:37.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:37.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:37.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:37.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:37.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/logs
08:03:37.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/data/info.pb
08:03:37.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:37.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:37.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:37.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:37.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:37.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:37.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:37.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:37.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:37.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:37.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:37.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:37.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.126:39261
08:03:37.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.126
08:03:37.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:37.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:37.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:37.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44381/jwks.json with env {}
08:03:38.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:38.146806 1473 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:38.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:38.147334 1473 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:38.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:38.147970 1473 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:38.180529 1473 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:38.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:38.180837 1473 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:38.181036 1473 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:38.181221 1473 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:38.181470 1473 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:38.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:38.181622 1473 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44381/jwks.json
08:03:38.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:38.217370 1473 master_runner.cc:387] Master server non-default flags:
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:38.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:38.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/data
08:03:38.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/wal
08:03:38.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:38.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:38.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:38.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:38.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:38.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.126:39261
08:03:38.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:38.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:38.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:38.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44381/jwks.json
08:03:38.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:38.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:38.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/data/info.pb
08:03:38.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.126
08:03:38.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:38.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:38.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/logs
08:03:38.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:38.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:38.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:38.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:38.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:38.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:03:38.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:03:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:03:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:38.218750 1473 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:38.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:38.220441 1473 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:38.232843 1479 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:39.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:38.233897 1480 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:39.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:39.447718 1473 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.216s user 0.492s sys 0.720s
08:03:39.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:39.448050 1473 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.216s user 0.492s sys 0.720s
08:03:39.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.450284 1473 server_base.cc:1061] running on GCE node
08:03:39.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:39.451494 1484 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:39.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.453037 1473 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:39.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.456249 1473 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.458204 1473 hybrid_clock.cc:648] HybridClock initialized: now 1781683419458158 us; error 35 us; skew 500 ppm
08:03:39.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.459434 1473 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:39.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.468773 1473 webserver.cc:533] Webserver started at http://127.1.109.126:33657/ using document root <none> and password file <none>
08:03:39.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.470219 1473 fs_manager.cc:362] Metadata directory not provided
08:03:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.470531 1473 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:39.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.471084 1473 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:39.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.478404 1473 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/data/instance:
08:03:39.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "766891ab88194a5289c4b8a05794685b"
08:03:39.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:39 on dist-test-slave-rgbw"
08:03:39.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.480026 1473 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/wal/instance:
08:03:39.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "766891ab88194a5289c4b8a05794685b"
08:03:39.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:39 on dist-test-slave-rgbw"
08:03:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.489115 1473 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
08:03:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.495180 1489 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:39.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.496538 1473 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
08:03:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.496955 1473 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/wal
08:03:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "766891ab88194a5289c4b8a05794685b"
08:03:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:39 on dist-test-slave-rgbw"
08:03:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.497412 1473 fs_report.cc:389] FS layout report
08:03:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/wal
08:03:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/wal
08:03:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/data/data
08:03:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:39.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:39.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:39.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:39.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:39.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:39.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:39.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.570546 1473 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:39.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.572278 1473 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.572829 1473 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:39.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.642743 1473 rpc_server.cc:307] RPC server started. Bound to: 127.1.109.126:39261
08:03:39.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.642856 1540 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.109.126:39261 every 8 connection(s)
08:03:39.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.645749 1473 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/data/info.pb
08:03:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.652683 1541 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.652597 1461 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 1473
08:03:39.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.653192 1461 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-0/wal/instance
08:03:39.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.658879 1461 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:39.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:39.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/wal
08:03:39.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/data
08:03:39.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:39.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:39.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/logs
08:03:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/data/info.pb
08:03:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:39.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:39.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.125:41681
08:03:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.125
08:03:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:39.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:39.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:39.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44381/jwks.json with env {}
08:03:39.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.668354 1541 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } has no permanent_uuid. Determining permanent_uuid...
08:03:39.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.700048 1541 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } has no permanent_uuid. Determining permanent_uuid...
08:03:39.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:39.702973 1492 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.109.125:41681: connect: Connection refused (error 111)
08:03:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:39.707688 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.125:41681: Network error: Client connection negotiation failed: client connection to 127.1.109.125:41681: connect: Connection refused (error 111)
08:03:39.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.757248 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } attempt: 1
08:03:39.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:39.762041 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.125:41681: Network error: Client connection negotiation failed: client connection to 127.1.109.125:41681: connect: Connection refused (error 111)
08:03:39.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.830539 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } attempt: 2
08:03:39.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:39.835511 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.125:41681: Network error: Client connection negotiation failed: client connection to 127.1.109.125:41681: connect: Connection refused (error 111)
08:03:39.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:39.926988 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } attempt: 3
08:03:39.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:39.933081 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.125:41681: Network error: Client connection negotiation failed: client connection to 127.1.109.125:41681: connect: Connection refused (error 111)
08:03:39.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:39.981801 1543 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:39.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:39.982359 1543 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:39.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:39.982805 1543 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:40.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:40.015008 1543 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:40.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:40.015290 1543 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:40.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:40.015479 1543 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:40.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:40.015805 1543 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:40.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:40.016132 1543 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:40.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:40.016382 1543 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44381/jwks.json
08:03:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:40.061560 1543 master_runner.cc:387] Master server non-default flags:
08:03:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:40.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:40.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:40.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/data
08:03:40.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/wal
08:03:40.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:40.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:40.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:40.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:40.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:40.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.125:41681
08:03:40.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:40.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:40.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:40.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44381/jwks.json
08:03:40.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:40.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:40.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/data/info.pb
08:03:40.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.125
08:03:40.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:40.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:40.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:40.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:40.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:40.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:40.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/logs
08:03:40.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:40.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:03:40.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:40.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:03:40.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:03:40.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:40.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:40.062903 1543 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:40.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:40.064582 1543 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:40.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:40.075832 1552 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:40.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:40.076517 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } attempt: 4
08:03:40.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:40.084362 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.125:41681: Network error: Client connection negotiation failed: client connection to 127.1.109.125:41681: connect: Connection refused (error 111)
08:03:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:40.383917 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } attempt: 5
08:03:40.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:40.393908 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.125:41681: Network error: Client connection negotiation failed: client connection to 127.1.109.125:41681: connect: Connection refused (error 111)
08:03:40.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:40.941459 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } attempt: 6
08:03:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:40.984964 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.125:41681: Network error: Client connection negotiation failed: client connection to 127.1.109.125:41681: connect: Connection refused (error 111)
08:03:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:41.480221 1551 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 1543
08:03:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:41.561384 1543 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.485s user 0.504s sys 0.972s
08:03:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:41.561791 1543 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.485s user 0.504s sys 0.972s
08:03:41.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:40.078102 1553 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:41.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:41.564539 1555 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:41.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:41.566684 1554 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1485 milliseconds
08:03:41.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.566790 1543 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:41.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.567987 1543 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:41.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.570325 1543 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:41.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.571660 1543 hybrid_clock.cc:648] HybridClock initialized: now 1781683421571634 us; error 39 us; skew 500 ppm
08:03:41.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.572459 1543 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:41.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.578446 1543 webserver.cc:533] Webserver started at http://127.1.109.125:41803/ using document root <none> and password file <none>
08:03:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.579613 1543 fs_manager.cc:362] Metadata directory not provided
08:03:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.579842 1543 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:41.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.580215 1543 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.584741 1543 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/data/instance:
08:03:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e202df1c396c449aa0f54342b15d2b08"
08:03:41.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:41 on dist-test-slave-rgbw"
08:03:41.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.585822 1543 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/wal/instance:
08:03:41.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e202df1c396c449aa0f54342b15d2b08"
08:03:41.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:41 on dist-test-slave-rgbw"
08:03:41.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.592463 1543 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.005s
08:03:41.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.597810 1564 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.598877 1543 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.005s
08:03:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.599138 1543 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/wal
08:03:41.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e202df1c396c449aa0f54342b15d2b08"
08:03:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:41 on dist-test-slave-rgbw"
08:03:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.599416 1543 fs_report.cc:389] FS layout report
08:03:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/wal
08:03:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/wal
08:03:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/data/data
08:03:41.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:41.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:41.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:41.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.657239 1543 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:41.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.658732 1543 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:41.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.659132 1543 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.728711 1543 rpc_server.cc:307] RPC server started. Bound to: 127.1.109.125:41681
08:03:41.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.728802 1615 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.109.125:41681 every 8 connection(s)
08:03:41.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.731580 1543 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/data/info.pb
08:03:41.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.738044 1616 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:41.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.740549 1461 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 1543
08:03:41.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.741034 1461 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-1/wal/instance
08:03:41.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.743247 1461 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:41.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:41.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/wal
08:03:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/data
08:03:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:41.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/logs
08:03:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/data/info.pb
08:03:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:41.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:41.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.124:33905
08:03:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.124
08:03:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:41.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:41.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:41.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:41.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44381/jwks.json with env {}
08:03:41.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.750552 1616 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } has no permanent_uuid. Determining permanent_uuid...
08:03:41.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.776384 1616 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } has no permanent_uuid. Determining permanent_uuid...
08:03:41.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.790544 1616 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } has no permanent_uuid. Determining permanent_uuid...
08:03:41.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:41.793315 1568 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:41.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:41.797900 1616 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:41.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.847362 1616 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 1
08:03:41.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:41.852548 1616 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:41.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:41.921016 1616 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 2
08:03:41.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:41.925545 1616 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:42.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.017005 1616 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 3
08:03:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.021677 1616 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:42.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.045521 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } attempt: 7
08:03:42.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.056847 1541 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } has no permanent_uuid. Determining permanent_uuid...
08:03:42.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.062425 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:42.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.064378 1618 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.064946 1618 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:42.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.065510 1618 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.098368 1618 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:42.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.098640 1618 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:42.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.098827 1618 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:42.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.099023 1618 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:42.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.099198 1618 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:42.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.099387 1618 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44381/jwks.json
08:03:42.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.120880 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 1
08:03:42.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.125566 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.138537 1618 master_runner.cc:387] Master server non-default flags:
08:03:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/data
08:03:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/wal
08:03:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:42.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:42.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:42.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:42.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.124:33905
08:03:42.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:42.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:42.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44381/jwks.json
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/data/info.pb
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.124
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:42.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/logs
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.140009 1618 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:42.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.142227 1618 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:42.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.154306 1630 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:42.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.165069 1616 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 4
08:03:42.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.172070 1616 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:42.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.207055 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 2
08:03:42.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.216090 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:42.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.301674 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 3
08:03:42.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.310671 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:42.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.451257 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 4
08:03:42.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.460536 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:42.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.471534 1616 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 5
08:03:42.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.480670 1616 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:42.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:42.744071 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 5
08:03:42.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.751740 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:43.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.028255 1616 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 6
08:03:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:43.056484 1616 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:43.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.304240 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 6
08:03:43.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:43.331182 1541 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.1.109.124:33905: Network error: Client connection negotiation failed: client connection to 127.1.109.124:33905: connect: Connection refused (error 111)
08:03:43.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:42.154790 1631 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:43.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:43.559199 1629 debug-util.cc:398] Leaking SignalData structure 0x7b0800028020 after lost signal to thread 1618
08:03:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:43.729753 1618 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.575s user 0.607s sys 0.956s
08:03:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:43.732095 1618 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.578s user 0.608s sys 0.957s
08:03:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:43.732271 1633 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:43.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:43.732844 1632 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1576 milliseconds
08:03:43.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.733036 1618 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:43.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.737134 1618 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:43.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.739645 1618 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:43.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.741046 1618 hybrid_clock.cc:648] HybridClock initialized: now 1781683423741003 us; error 47 us; skew 500 ppm
08:03:43.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.742008 1618 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:43.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.748658 1618 webserver.cc:533] Webserver started at http://127.1.109.124:43007/ using document root <none> and password file <none>
08:03:43.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.749720 1618 fs_manager.cc:362] Metadata directory not provided
08:03:43.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.749917 1618 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:43.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.750540 1618 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:43.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.755043 1618 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/data/instance:
08:03:43.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b3d46f27e1b48068aceaa196c547855"
08:03:43.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:43 on dist-test-slave-rgbw"
08:03:43.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.756079 1618 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/wal/instance:
08:03:43.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b3d46f27e1b48068aceaa196c547855"
08:03:43.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:43 on dist-test-slave-rgbw"
08:03:43.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.763010 1618 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.009s sys 0.000s
08:03:43.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.769174 1644 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:43.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.770382 1618 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
08:03:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.770697 1618 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/wal
08:03:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3b3d46f27e1b48068aceaa196c547855"
08:03:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:43 on dist-test-slave-rgbw"
08:03:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.771030 1618 fs_report.cc:389] FS layout report
08:03:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/wal
08:03:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/wal
08:03:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/data/data
08:03:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:43.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:43.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:43.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:43.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:43.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:43.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:43.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:43.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:43.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:43.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:43.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.821837 1618 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.823237 1618 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:43.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.823623 1618 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.895782 1618 rpc_server.cc:307] RPC server started. Bound to: 127.1.109.124:33905
08:03:43.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.895855 1695 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.109.124:33905 every 8 connection(s)
08:03:43.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.898689 1618 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/data/info.pb
08:03:43.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.903542 1461 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 1618
08:03:43.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.904052 1461 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/master-2/wal/instance
08:03:43.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.906168 1696 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:43.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.925938 1696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } has no permanent_uuid. Determining permanent_uuid...
08:03:43.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.950453 1696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } has no permanent_uuid. Determining permanent_uuid...
08:03:43.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.960172 1696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } has no permanent_uuid. Determining permanent_uuid...
08:03:43.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.976332 1696 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855: Bootstrap starting.
08:03:43.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.985775 1696 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855: Neither blocks nor log segments found. Creating new log.
08:03:43.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.988590 1696 log.cc:826] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855: Log is configured to *not* fsync() on all Append() calls
08:03:43.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:43.993826 1696 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855: No bootstrap required, opened a new log
08:03:44.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.012447 1696 raft_consensus.cc:359] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } }
08:03:44.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.013088 1696 raft_consensus.cc:385] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:44.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.013274 1696 raft_consensus.cc:740] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3b3d46f27e1b48068aceaa196c547855, State: Initialized, Role: FOLLOWER
08:03:44.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.013934 1696 consensus_queue.cc:260] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } }
08:03:44.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.016140 1705 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } } }
08:03:44.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.016971 1705 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [sys.catalog]: This master's current role is: FOLLOWER
08:03:44.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.017735 1696 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [sys.catalog]: configured and running, proceeding with master startup.
08:03:44.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:44.042435 1716 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:44.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:44.042713 1716 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:44.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.039175 1618 master_runner.cc:187] Error getting master registration for 127.1.109.126:39261: OK, instance_id { permanent_uuid: "766891ab88194a5289c4b8a05794685b" instance_seqno: 1781683419623546 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:44.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.062697 1618 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:03:44.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:44.079365 1650 tablet.cc:2406] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:03:44.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.117074 1616 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 7
08:03:44.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.135095 1616 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08: Bootstrap starting.
08:03:44.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.141285 1616 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08: Neither blocks nor log segments found. Creating new log.
08:03:44.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.144164 1616 log.cc:826] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08: Log is configured to *not* fsync() on all Append() calls
08:03:44.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.148561 1616 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08: No bootstrap required, opened a new log
08:03:44.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.167984 1616 raft_consensus.cc:359] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } }
08:03:44.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.168632 1616 raft_consensus.cc:385] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:44.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.168910 1616 raft_consensus.cc:740] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e202df1c396c449aa0f54342b15d2b08, State: Initialized, Role: FOLLOWER
08:03:44.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.169808 1616 consensus_queue.cc:260] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } }
08:03:44.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.172439 1720 sys_catalog.cc:455] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } } }
08:03:44.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.173259 1720 sys_catalog.cc:458] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [sys.catalog]: This master's current role is: FOLLOWER
08:03:44.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.174372 1616 sys_catalog.cc:565] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [sys.catalog]: configured and running, proceeding with master startup.
08:03:44.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:44.208822 1731 catalog_manager.cc:1593] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:03:44.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:44.209714 1731 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:03:44.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.215708 1705 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:03:44.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.216243 1705 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } }
08:03:44.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.220220 1591 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3b3d46f27e1b48068aceaa196c547855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e202df1c396c449aa0f54342b15d2b08" is_pre_election: true
08:03:44.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.221025 1591 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3b3d46f27e1b48068aceaa196c547855 in term 0.
08:03:44.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.221467 1705 leader_election.cc:290] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 766891ab88194a5289c4b8a05794685b (127.1.109.126:39261), e202df1c396c449aa0f54342b15d2b08 (127.1.109.125:41681)
08:03:44.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.222580 1647 leader_election.cc:304] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3b3d46f27e1b48068aceaa196c547855, e202df1c396c449aa0f54342b15d2b08; no voters:
08:03:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.223587 1705 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [term 0 FOLLOWER]: Leader pre-election won for term 1
08:03:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.223901 1705 raft_consensus.cc:493] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:03:44.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.224241 1705 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [term 0 FOLLOWER]: Advancing to term 1
08:03:44.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.232189 1705 raft_consensus.cc:515] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } }
08:03:44.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.239060 1591 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3b3d46f27e1b48068aceaa196c547855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e202df1c396c449aa0f54342b15d2b08"
08:03:44.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.239612 1591 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [term 0 FOLLOWER]: Advancing to term 1
08:03:44.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.246618 1591 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3b3d46f27e1b48068aceaa196c547855 in term 1.
08:03:44.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.248131 1647 leader_election.cc:304] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 3b3d46f27e1b48068aceaa196c547855, e202df1c396c449aa0f54342b15d2b08; no voters:
08:03:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.253783 1705 leader_election.cc:290] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [CANDIDATE]: Term 1 election: Requested vote from peers 766891ab88194a5289c4b8a05794685b (127.1.109.126:39261), e202df1c396c449aa0f54342b15d2b08 (127.1.109.125:41681)
08:03:44.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.254473 1705 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [term 1 FOLLOWER]: Leader election won for term 1
08:03:44.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.254904 1705 raft_consensus.cc:697] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [term 1 LEADER]: Becoming Leader. State: Replica: 3b3d46f27e1b48068aceaa196c547855, State: Running, Role: LEADER
08:03:44.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.255791 1705 consensus_queue.cc:237] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } }
08:03:44.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.201117 1543 master_runner.cc:187] Error getting master registration for 127.1.109.126:39261: OK, instance_id { permanent_uuid: "766891ab88194a5289c4b8a05794685b" instance_seqno: 1781683419623546 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:03:44.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.261785 1543 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:03:44.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.273425 1733 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3b3d46f27e1b48068aceaa196c547855. Latest consensus state: current_term: 1 leader_uuid: "3b3d46f27e1b48068aceaa196c547855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } } }
08:03:44.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.274544 1733 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [sys.catalog]: This master's current role is: LEADER
08:03:44.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.282230 1735 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
08:03:44.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.289433 1735 catalog_manager.cc:1519] Initializing Kudu cluster ID...
08:03:44.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.314113 1591 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [term 1 FOLLOWER]: Refusing update from remote peer 3b3d46f27e1b48068aceaa196c547855: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:44.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.315793 1733 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:44.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.345367 1720 sys_catalog.cc:455] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3b3d46f27e1b48068aceaa196c547855. Latest consensus state: current_term: 1 leader_uuid: "3b3d46f27e1b48068aceaa196c547855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } } }
08:03:44.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.346103 1720 sys_catalog.cc:458] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [sys.catalog]: This master's current role is: FOLLOWER
08:03:44.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.354892 1736 mvcc.cc:204] Tried to move back new op lower bound from 7297775305965555712 to 7297775305774096384. Current Snapshot: MvccSnapshot[applied={T|T < 7297775305965555712}]
08:03:44.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.360019 1720 sys_catalog.cc:455] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3b3d46f27e1b48068aceaa196c547855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } } }
08:03:44.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.361207 1720 sys_catalog.cc:458] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08 [sys.catalog]: This master's current role is: FOLLOWER
08:03:44.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.364696 1541 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } attempt: 7
08:03:44.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.371104 1733 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3b3d46f27e1b48068aceaa196c547855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } } }
08:03:44.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.372059 1733 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [sys.catalog]: This master's current role is: LEADER
08:03:44.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.375293 1735 catalog_manager.cc:1382] Generated new cluster ID: 43b38ccf7dd7431c9ed28b892a077c13
08:03:44.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.375573 1735 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
08:03:44.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.414956 1541 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b: Bootstrap starting.
08:03:44.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.423894 1735 catalog_manager.cc:1405] Generated new certificate authority record
08:03:44.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.427182 1735 catalog_manager.cc:1539] Loading token signing keys...
08:03:44.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.438099 1541 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b: Neither blocks nor log segments found. Creating new log.
08:03:44.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.440168 1541 log.cc:826] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b: Log is configured to *not* fsync() on all Append() calls
08:03:44.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.451927 1541 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b: No bootstrap required, opened a new log
08:03:44.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.475718 1735 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855: Generated new TSK 0
08:03:44.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.477530 1735 catalog_manager.cc:1549] Initializing in-progress tserver states...
08:03:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.494915 1461 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/wal
08:03:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/data
08:03:44.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/logs
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/data/info.pb
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:44.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.65:0
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.109.65
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.65
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:44.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.511232 1541 raft_consensus.cc:359] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } }
08:03:44.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.513309 1541 raft_consensus.cc:385] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:03:44.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.513916 1541 raft_consensus.cc:740] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 766891ab88194a5289c4b8a05794685b, State: Initialized, Role: FOLLOWER
08:03:44.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.515182 1541 consensus_queue.cc:260] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } }
08:03:44.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.524057 1744 sys_catalog.cc:455] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } } }
08:03:44.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.527979 1744 sys_catalog.cc:458] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [sys.catalog]: This master's current role is: FOLLOWER
08:03:44.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.528885 1515 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3b3d46f27e1b48068aceaa196c547855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "766891ab88194a5289c4b8a05794685b"
08:03:44.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.529440 1514 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [term 0 FOLLOWER]: Advancing to term 1
08:03:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.527776 1516 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "3b3d46f27e1b48068aceaa196c547855" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "766891ab88194a5289c4b8a05794685b" is_pre_election: true
08:03:44.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.527596 1541 sys_catalog.cc:565] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [sys.catalog]: configured and running, proceeding with master startup.
08:03:44.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.539546 1514 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [term 1 FOLLOWER]: Refusing update from remote peer 3b3d46f27e1b48068aceaa196c547855: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:03:44.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.541450 1733 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [LEADER]: Connected to new peer: Peer: permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:03:44.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.618528 1744 sys_catalog.cc:455] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [sys.catalog]: SysCatalogTable state changed. Reason: New leader 3b3d46f27e1b48068aceaa196c547855. Latest consensus state: current_term: 1 leader_uuid: "3b3d46f27e1b48068aceaa196c547855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } } }
08:03:44.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.618745 1749 sys_catalog.cc:455] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "3b3d46f27e1b48068aceaa196c547855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } } }
08:03:44.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.620318 1749 sys_catalog.cc:458] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [sys.catalog]: This master's current role is: FOLLOWER
08:03:44.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.620551 1744 sys_catalog.cc:458] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b [sys.catalog]: This master's current role is: FOLLOWER
08:03:44.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.679271 1760 catalog_manager.cc:1294] Loaded cluster ID: 43b38ccf7dd7431c9ed28b892a077c13
08:03:44.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.692848 1760 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b: loading cluster ID for follower catalog manager: success
08:03:44.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.708724 1760 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b: acquiring CA information for follower catalog manager: success
08:03:44.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:44.722486 1760 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 766891ab88194a5289c4b8a05794685b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:45.190361 1762 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "3b3d46f27e1b48068aceaa196c547855" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "766891ab88194a5289c4b8a05794685b" member_type: VOTER last_known_addr { host: "127.1.109.126" port: 39261 } } peers { permanent_uuid: "e202df1c396c449aa0f54342b15d2b08" member_type: VOTER last_known_addr { host: "127.1.109.125" port: 41681 } } peers { permanent_uuid: "3b3d46f27e1b48068aceaa196c547855" member_type: VOTER last_known_addr { host: "127.1.109.124" port: 33905 } } }
08:03:45.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:45.205560 1762 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3b3d46f27e1b48068aceaa196c547855 [sys.catalog]: This master's current role is: LEADER
08:03:45.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:45.215803 1731 catalog_manager.cc:1294] Loaded cluster ID: 43b38ccf7dd7431c9ed28b892a077c13
08:03:45.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:45.216094 1731 catalog_manager.cc:1587] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08: loading cluster ID for follower catalog manager: success
08:03:45.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:45.223038 1731 catalog_manager.cc:1609] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08: acquiring CA information for follower catalog manager: success
08:03:45.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:45.238837 1731 catalog_manager.cc:1637] T 00000000000000000000000000000000 P e202df1c396c449aa0f54342b15d2b08: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:03:45.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:45.266460 1743 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:45.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:45.267570 1743 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:45.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:45.268543 1743 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:45.329509 1743 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:45.331264 1743 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.109.65
08:03:45.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:45.401317 1743 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/data
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/wal
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.65:0
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:45.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/data/info.pb
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.65
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.109.65
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/logs
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:45.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:45.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:03:45.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:45.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:03:45.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:03:45.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:45.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:45.411788 1743 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:45.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:45.414487 1743 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:45.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:45.452986 1770 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:46.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:45.460906 1771 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:46.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:45.465757 1774 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:46.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:45.466058 1743 server_base.cc:1061] running on GCE node
08:03:46.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:46.992678 1743 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:47.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.002712 1743 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:47.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.004392 1743 hybrid_clock.cc:648] HybridClock initialized: now 1781683427004326 us; error 78 us; skew 500 ppm
08:03:47.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.005681 1743 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:47.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.049510 1743 webserver.cc:533] Webserver started at http://127.1.109.65:43615/ using document root <none> and password file <none>
08:03:47.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.051605 1743 fs_manager.cc:362] Metadata directory not provided
08:03:47.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.052143 1743 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:47.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.052923 1743 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:47.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.059535 1743 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/data/instance:
08:03:47.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbdce5eb0395465ba7dcdfc7ea3c5ce6"
08:03:47.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:47 on dist-test-slave-rgbw"
08:03:47.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.061460 1743 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/wal/instance:
08:03:47.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbdce5eb0395465ba7dcdfc7ea3c5ce6"
08:03:47.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:47 on dist-test-slave-rgbw"
08:03:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.076574 1743 fs_manager.cc:696] Time spent creating directory manager: real 0.014s user 0.008s sys 0.004s
08:03:47.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.091580 1780 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:47.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.093185 1743 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.004s sys 0.004s
08:03:47.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.093819 1743 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/wal
08:03:47.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "cbdce5eb0395465ba7dcdfc7ea3c5ce6"
08:03:47.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:47 on dist-test-slave-rgbw"
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.094563 1743 fs_report.cc:389] FS layout report
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/wal
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/wal
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/data/data
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:47.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:47.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:47.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:47.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.198325 1743 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:47.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.200600 1743 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:47.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.201478 1743 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:47.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.210878 1743 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.225980 1743 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.226647 1743 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
08:03:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.236970 1743 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.237550 1743 ts_tablet_manager.cc:595] Time spent register tablets: real 0.001s user 0.000s sys 0.000s
08:03:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.485087 1743 rpc_server.cc:307] RPC server started. Bound to: 127.1.109.65:46299
08:03:47.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.485184 1892 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.109.65:46299 every 8 connection(s)
08:03:47.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.488540 1743 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/data/info.pb
08:03:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.497819 1461 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 1743
08:03:47.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.498400 1461 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-0/wal/instance
08:03:47.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.516161 1461 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:47.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/wal
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/data
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/logs
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/data/info.pb
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:47.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.66:0
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.109.66
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.66
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:47.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:47.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.595094 1898 heartbeater.cc:344] Connected to a master server at 127.1.109.126:39261
08:03:47.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.595598 1898 heartbeater.cc:461] Registering TS with master...
08:03:47.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.596881 1898 heartbeater.cc:507] Master 127.1.109.126:39261 requested a full tablet report, sending...
08:03:47.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.601279 1894 heartbeater.cc:344] Connected to a master server at 127.1.109.125:41681
08:03:47.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.601188 1506 ts_manager.cc:194] Registered new tserver with Master: cbdce5eb0395465ba7dcdfc7ea3c5ce6 (127.1.109.65:46299)
08:03:47.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.601598 1894 heartbeater.cc:461] Registering TS with master...
08:03:47.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.602448 1894 heartbeater.cc:507] Master 127.1.109.125:41681 requested a full tablet report, sending...
08:03:47.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.605906 1581 ts_manager.cc:194] Registered new tserver with Master: cbdce5eb0395465ba7dcdfc7ea3c5ce6 (127.1.109.65:46299)
08:03:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.609802 1893 heartbeater.cc:344] Connected to a master server at 127.1.109.124:33905
08:03:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.610143 1893 heartbeater.cc:461] Registering TS with master...
08:03:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.611014 1893 heartbeater.cc:507] Master 127.1.109.124:33905 requested a full tablet report, sending...
08:03:47.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.613633 1661 ts_manager.cc:194] Registered new tserver with Master: cbdce5eb0395465ba7dcdfc7ea3c5ce6 (127.1.109.65:46299)
08:03:47.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:47.616250 1661 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.109.65:54183
08:03:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:47.990505 1897 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:47.990986 1897 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:47.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:47.991465 1897 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:48.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:48.024493 1897 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:48.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:48.025333 1897 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.109.66
08:03:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:48.073680 1897 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/data
08:03:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/wal
08:03:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.66:0
08:03:48.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:48.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/data/info.pb
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.66
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.109.66
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/logs
08:03:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:48.075124 1897 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:48.077069 1897 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:48.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:48.095448 1912 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:48.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:48.621608 1893 heartbeater.cc:499] Master 127.1.109.124:33905 was elected leader, sending a full tablet report...
08:03:49.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:48.095633 1911 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:49.338900 1897 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.246s user 0.422s sys 0.822s
08:03:49.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:49.339277 1897 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.247s user 0.422s sys 0.822s
08:03:49.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.343057 1897 server_base.cc:1061] running on GCE node
08:03:49.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:49.343636 1917 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:49.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.345157 1897 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:49.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.348148 1897 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.352922 1897 hybrid_clock.cc:648] HybridClock initialized: now 1781683429352839 us; error 75 us; skew 500 ppm
08:03:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.353801 1897 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:49.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.365644 1897 webserver.cc:533] Webserver started at http://127.1.109.66:37211/ using document root <none> and password file <none>
08:03:49.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.366611 1897 fs_manager.cc:362] Metadata directory not provided
08:03:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.366859 1897 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:49.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.367372 1897 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:49.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.372218 1897 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/data/instance:
08:03:49.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "906a0230b62e48bd8b678409f5532024"
08:03:49.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:49 on dist-test-slave-rgbw"
08:03:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.373809 1897 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/wal/instance:
08:03:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "906a0230b62e48bd8b678409f5532024"
08:03:49.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:49 on dist-test-slave-rgbw"
08:03:49.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.382124 1897 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.002s
08:03:49.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.391765 1922 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:49.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.393287 1897 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.002s sys 0.003s
08:03:49.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.393954 1897 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/wal
08:03:49.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "906a0230b62e48bd8b678409f5532024"
08:03:49.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:49 on dist-test-slave-rgbw"
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.394315 1897 fs_report.cc:389] FS layout report
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/wal
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/wal
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/data/data
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:49.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:49.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:49.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:49.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.488011 1897 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:49.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.489599 1897 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:49.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.490127 1897 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:49.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.492825 1897 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:49.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.497295 1897 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:49.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.497516 1897 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:49.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.497866 1897 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:49.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.498066 1897 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:49.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.646915 1897 rpc_server.cc:307] RPC server started. Bound to: 127.1.109.66:40169
08:03:49.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.647063 2035 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.109.66:40169 every 8 connection(s)
08:03:49.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.649678 1897 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/data/info.pb
08:03:49.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.658712 1461 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 1897
08:03:49.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.659462 1461 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-1/wal/instance
08:03:49.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.674288 1461 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/wal
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/data
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/logs
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/data/info.pb
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:49.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.67:0
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.109.67
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.67
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:49.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:49.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
08:03:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.710804 2036 heartbeater.cc:344] Connected to a master server at 127.1.109.124:33905
08:03:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.711293 2036 heartbeater.cc:461] Registering TS with master...
08:03:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.712531 2036 heartbeater.cc:507] Master 127.1.109.124:33905 requested a full tablet report, sending...
08:03:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.713379 2037 heartbeater.cc:344] Connected to a master server at 127.1.109.125:41681
08:03:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.713728 2037 heartbeater.cc:461] Registering TS with master...
08:03:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.714548 2037 heartbeater.cc:507] Master 127.1.109.125:41681 requested a full tablet report, sending...
08:03:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.715572 2038 heartbeater.cc:344] Connected to a master server at 127.1.109.126:39261
08:03:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.715900 2038 heartbeater.cc:461] Registering TS with master...
08:03:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.716446 1661 ts_manager.cc:194] Registered new tserver with Master: 906a0230b62e48bd8b678409f5532024 (127.1.109.66:40169)
08:03:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.716622 2038 heartbeater.cc:507] Master 127.1.109.126:39261 requested a full tablet report, sending...
08:03:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.718278 1581 ts_manager.cc:194] Registered new tserver with Master: 906a0230b62e48bd8b678409f5532024 (127.1.109.66:40169)
08:03:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.718390 1661 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.109.66:40397
08:03:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:49.719650 1506 ts_manager.cc:194] Registered new tserver with Master: 906a0230b62e48bd8b678409f5532024 (127.1.109.66:40169)
08:03:50.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:50.025087 2046 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:50.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:50.025606 2046 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:50.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:50.026155 2046 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:50.062657 2046 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:50.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:50.063613 2046 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.1.109.67
08:03:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:50.100871 2046 tablet_server_runner.cc:78] Tablet server non-default flags:
08:03:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.1.109.84:34745
08:03:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/data
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/wal
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.1.109.67:0
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/data/info.pb
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.1.109.67
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.1.109.126:39261,127.1.109.125:41681,127.1.109.124:33905
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.1.109.67
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/logs
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:50.102581 2046 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:50.104686 2046 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:50.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:50.117946 2053 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:50.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:50.722390 2036 heartbeater.cc:499] Master 127.1.109.124:33905 was elected leader, sending a full tablet report...
08:03:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:51.521889 2052 debug-util.cc:398] Leaking SignalData structure 0x7b0800028000 after lost signal to thread 2046
08:03:51.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:51.609998 2046 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.492s user 0.528s sys 0.915s
08:03:51.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:51.610313 2046 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.493s user 0.528s sys 0.915s
08:03:51.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:50.118551 2054 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:51.612882 2056 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:51.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:51.616840 2055 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1498 milliseconds
08:03:51.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.616854 2046 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:51.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.618098 2046 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:51.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.620191 2046 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.621547 2046 hybrid_clock.cc:648] HybridClock initialized: now 1781683431621514 us; error 41 us; skew 500 ppm
08:03:51.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.622396 2046 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:51.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.630110 2046 webserver.cc:533] Webserver started at http://127.1.109.67:34971/ using document root <none> and password file <none>
08:03:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.631136 2046 fs_manager.cc:362] Metadata directory not provided
08:03:51.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.631368 2046 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:51.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.631894 2046 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:51.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.636427 2046 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/data/instance:
08:03:51.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a960d49f8b8d4e1db9ef78cec9c78d7c"
08:03:51.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:51 on dist-test-slave-rgbw"
08:03:51.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.637641 2046 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/wal/instance:
08:03:51.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a960d49f8b8d4e1db9ef78cec9c78d7c"
08:03:51.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:51 on dist-test-slave-rgbw"
08:03:51.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.646548 2046 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
08:03:51.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.654182 2064 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:51.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.655646 2046 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
08:03:51.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.656229 2046 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/wal
08:03:51.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a960d49f8b8d4e1db9ef78cec9c78d7c"
08:03:51.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:51 on dist-test-slave-rgbw"
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.656635 2046 fs_report.cc:389] FS layout report
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/wal
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/wal
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/data/data
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:51.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:51.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:51.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:51.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:51.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.714442 2046 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:51.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.715858 2046 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:51.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.716248 2046 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:51.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.719380 2046 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:03:51.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.723383 2046 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:03:51.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.723567 2046 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:03:51.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.723786 2046 ts_tablet_manager.cc:616] Registered 0 tablets
08:03:51.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.723982 2046 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:03:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.877971 2046 rpc_server.cc:307] RPC server started. Bound to: 127.1.109.67:44281
08:03:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.878063 2176 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.1.109.67:44281 every 8 connection(s)
08:03:51.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.880618 2046 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/data/info.pb
08:03:51.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.888191 1461 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 2046
08:03:51.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.888651 1461 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7363293730725449819/ts-2/wal/instance
08:03:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.926088 2178 heartbeater.cc:344] Connected to a master server at 127.1.109.125:41681
08:03:51.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.927069 2178 heartbeater.cc:461] Registering TS with master...
08:03:51.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.928283 2178 heartbeater.cc:507] Master 127.1.109.125:41681 requested a full tablet report, sending...
08:03:51.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.931640 1581 ts_manager.cc:194] Registered new tserver with Master: a960d49f8b8d4e1db9ef78cec9c78d7c (127.1.109.67:44281)
08:03:51.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.934052 2177 heartbeater.cc:344] Connected to a master server at 127.1.109.124:33905
08:03:51.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.934351 2177 heartbeater.cc:461] Registering TS with master...
08:03:51.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.935242 2177 heartbeater.cc:507] Master 127.1.109.124:33905 requested a full tablet report, sending...
08:03:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.937562 1661 ts_manager.cc:194] Registered new tserver with Master: a960d49f8b8d4e1db9ef78cec9c78d7c (127.1.109.67:44281)
08:03:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.938220 2179 heartbeater.cc:344] Connected to a master server at 127.1.109.126:39261
08:03:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.938526 2179 heartbeater.cc:461] Registering TS with master...
08:03:51.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.939224 1661 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.1.109.67:33731
08:03:51.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.939312 2179 heartbeater.cc:507] Master 127.1.109.126:39261 requested a full tablet report, sending...
08:03:51.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.942652 1505 ts_manager.cc:194] Registered new tserver with Master: a960d49f8b8d4e1db9ef78cec9c78d7c (127.1.109.67:44281)
08:03:51.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:51.945315 1461 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
08:03:51.958 [DEBUG - main] (MiniKuduCluster.java:195) Response:
08:03:51.959 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
08:03:51.969 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.1.109.126"
port: 39261
}
bound_http_address {
host: "127.1.109.126"
port: 33657
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.1.109.125"
port: 41681
}
bound_http_address {
host: "127.1.109.125"
port: 41803
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.1.109.124"
port: 33905
}
bound_http_address {
host: "127.1.109.124"
port: 43007
}
}
}
08:03:51.972 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:03:51.975 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.1.109.65"
port: 46299
}
bound_http_address {
host: "127.1.109.65"
port: 43615
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.1.109.66"
port: 40169
}
bound_http_address {
host: "127.1.109.66"
port: 37211
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.1.109.67"
port: 44281
}
bound_http_address {
host: "127.1.109.67"
port: 34971
}
}
}
08:03:51.975 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
08:03:52.154 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
08:03:52.215 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
08:03:52.304 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskexLxKm/test-tmp
08:03:52.305 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
08:03:52.305 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskexLxKm/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
08:03:52.316 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
08:03:52.660 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:03:52.660 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
08:03:52.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:52.664922 2190 mini_oidc.cc:150] Starting JWKS server
08:03:52.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:52.668018 2190 webserver.cc:533] Webserver started at http://127.0.0.1:39839/ using document root <none> and password file <none>
08:03:52.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:52.668421 2190 mini_oidc.cc:186] Starting OIDC Discovery server
08:03:52.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:52.670871 2190 webserver.cc:533] Webserver started at http://127.0.0.1:35563/ using document root <none> and password file <none>
08:03:52.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
08:03:52.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/principal' for realm 'KRBTEST.COM',
08:03:52.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
08:03:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:03:52 dist-test-slave-rgbw krb5kdc[2202](info): setting up network...
08:03:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
08:03:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:03:52 dist-test-slave-rgbw krb5kdc[2202](info): set up 2 sockets
08:03:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:03:52 dist-test-slave-rgbw krb5kdc[2202](info): commencing operation
08:03:52.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
08:03:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:52.943161 2177 heartbeater.cc:499] Master 127.1.109.124:33905 was elected leader, sending a full tablet report...
08:03:54.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:54.807804 2190 mini_kdc.cc:121] Time spent starting KDC: real 2.097s user 0.006s sys 0.012s
08:03:54.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
08:03:54.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:03:54.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
08:03:54.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
08:03:54.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:03:54.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
08:03:54.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
08:03:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:03:54.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
08:03:54.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
08:03:54.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/test-user.keytab.
08:03:54.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/test-user.keytab.
08:03:54.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:03:54 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683434, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:03:54.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:03:54.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T08:03:54Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
08:03:54.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T08:03:54Z Disabled control of system clock
08:03:54.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.35.190@KRBTEST.COM; defaulting to no policy
08:03:54.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:03:54.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.35.190@KRBTEST.COM" created.
08:03:55.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:03:55.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:03:55.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:03:55.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.35.190@KRBTEST.COM; defaulting to no policy
08:03:55.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:03:55.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.35.190@KRBTEST.COM" created.
08:03:55.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:03:55.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:03:55.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:03:55.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:55.054294 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:55.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:55.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:03:55.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/logs
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.190:32797
08:03:55.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.190
08:03:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:03:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.190
08:03:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:03:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:03:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:03:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:03:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:03:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:55.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:55.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:55.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:55.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:03:55.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:55.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:03:55.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:03:55.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:03:55.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.366684 2218 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:55.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.367231 2218 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:55.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.367790 2218 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:55.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.401875 2218 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:03:55.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.402109 2218 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:55.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.402387 2218 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:55.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.402667 2218 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:55.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.402904 2218 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:55.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.403169 2218 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:55.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.403477 2218 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:03:55.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:55.445420 2218 master_runner.cc:387] Master server non-default flags:
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.190
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.190:32797
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:03:55.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.190
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/logs
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:55.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:55.446877 2218 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:55.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:55.448025 2218 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:55.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.460112 2226 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:56.862502 2225 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 2218
08:03:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:55.461607 2227 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.279052 2225 kernel_stack_watchdog.cc:198] Thread 2218 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
08:03:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:03:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:03:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:03:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:03:57.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.280349 2218 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.820s user 0.000s sys 0.002s
08:03:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.280797 2218 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.820s user 0.000s sys 0.002s
08:03:57.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.284356 2228 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1817 milliseconds
08:03:57.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.285760 2229 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:57.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.285869 2218 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:57.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.287199 2218 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:57.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.290073 2218 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:57.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.291486 2218 hybrid_clock.cc:648] HybridClock initialized: now 1781683437291437 us; error 52 us; skew 500 ppm
08:03:57.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.292486 2218 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:57.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:03:57 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683437, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:03:57.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.298144 2218 init.cc:377] Logged in from keytab as kudu/127.2.35.190@KRBTEST.COM (short username kudu)
08:03:57.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.303864 2218 webserver.cc:533] Webserver started at http://127.2.35.190:35165/ using document root <none> and password file <none>
08:03:57.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.304836 2218 fs_manager.cc:362] Metadata directory not provided
08:03:57.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.305078 2218 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:57.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.305585 2218 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:57.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.310137 2218 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/instance:
08:03:57.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fc933c428a74e04993ae685149e88d5"
08:03:57.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:57 on dist-test-slave-rgbw"
08:03:57.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.311261 2218 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal/instance:
08:03:57.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fc933c428a74e04993ae685149e88d5"
08:03:57.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:57 on dist-test-slave-rgbw"
08:03:57.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.318481 2218 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
08:03:57.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.324465 2239 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:57.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.325582 2218 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
08:03:57.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.325968 2218 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:03:57.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fc933c428a74e04993ae685149e88d5"
08:03:57.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:57 on dist-test-slave-rgbw"
08:03:57.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.326370 2218 fs_report.cc:389] FS layout report
08:03:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:03:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:03:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/data
08:03:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:57.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:57.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:57.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.413772 2218 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:57.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.419404 2218 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:57.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.419914 2218 kserver.cc:163] Server-wide thread pool size limit: 3276
08:03:57.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.502465 2218 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.190:32797
08:03:57.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.502573 2290 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.190:32797 every 8 connection(s)
08:03:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.505292 2218 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:03:57.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.512598 2291 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:03:57.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.514061 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 2218
08:03:57.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.514590 2190 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal/instance
08:03:57.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.528812 2291 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } has no permanent_uuid. Determining permanent_uuid...
08:03:57.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.35.189@KRBTEST.COM; defaulting to no policy
08:03:57.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:03:57.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.35.189@KRBTEST.COM" created.
08:03:57.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:03:57 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683437, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:03:57.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:03:57.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:03:57.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:03:57.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.567876 2291 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } has no permanent_uuid. Determining permanent_uuid...
08:03:57.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.570956 2241 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:03:57.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.576155 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.189:40817: Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:03:57.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.35.189@KRBTEST.COM; defaulting to no policy
08:03:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:03:57.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.35.189@KRBTEST.COM" created.
08:03:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:03:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:03:57.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.607304 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/logs
08:03:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:03:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:03:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:03:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:03:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:03:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:57.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.189:40817
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.189
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.189
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:03:57.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:03:57.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.625655 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } attempt: 1
08:03:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.630437 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.189:40817: Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:03:57.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.698838 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } attempt: 2
08:03:57.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.703680 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.189:40817: Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:03:57.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.795079 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } attempt: 3
08:03:57.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.799396 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.189:40817: Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:03:57.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:57.942792 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } attempt: 4
08:03:57.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.947044 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.189:40817: Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:03:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.960214 2300 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:03:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.960728 2300 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:03:57.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:57.961655 2300 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:03:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.000193 2300 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:03:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.000423 2300 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:03:58.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.000681 2300 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:03:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.000983 2300 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:03:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.001191 2300 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:03:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.001379 2300 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:03:58.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.001533 2300 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:58.040074 2300 master_runner.cc:387] Master server non-default flags:
08:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data
08:03:58.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.189
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.189:40817
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.189
08:03:58.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/logs
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:58.041414 2300 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:03:58.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:58.042613 2300 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:03:58.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.053821 2308 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:58.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:58.246454 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } attempt: 5
08:03:58.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.256126 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.189:40817: Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:03:58.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:58.803634 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } attempt: 6
08:03:58.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.825691 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.189:40817: Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:03:58.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.903327 1692 debug-util.cc:398] Leaking SignalData structure 0x7b0800074f40 after lost signal to thread 1623
08:03:58.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.904275 1692 debug-util.cc:398] Leaking SignalData structure 0x7b080009a6a0 after lost signal to thread 1695
08:03:59.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:59.456849 2307 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 2300
08:03:59.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:59.851291 2300 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.798s user 0.582s sys 1.133s
08:03:59.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:59.851651 2300 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.798s user 0.582s sys 1.133s
08:03:59.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:58.054637 2309 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:59.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:59.853056 2310 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1798 milliseconds
08:03:59.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.853965 2300 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:03:59.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:59.854015 2311 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:03:59.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.859503 2300 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:03:59.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.862625 2300 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:03:59.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.866603 2300 hybrid_clock.cc:648] HybridClock initialized: now 1781683439866560 us; error 43 us; skew 500 ppm
08:03:59.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.867507 2300 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:03:59.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:03:59 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683439, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:03:59.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.874104 2300 init.cc:377] Logged in from keytab as kudu/127.2.35.189@KRBTEST.COM (short username kudu)
08:03:59.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.880528 2300 webserver.cc:533] Webserver started at http://127.2.35.189:45919/ using document root <none> and password file <none>
08:03:59.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.881760 2300 fs_manager.cc:362] Metadata directory not provided
08:03:59.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.882045 2300 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:03:59.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.882606 2300 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:03:59.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.886312 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } attempt: 7
08:03:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.889616 2300 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/instance:
08:03:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91200c15660b4e97b54ebd3774fbcc1c"
08:03:59.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:59 on dist-test-slave-rgbw"
08:03:59.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.891512 2300 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal/instance:
08:03:59.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91200c15660b4e97b54ebd3774fbcc1c"
08:03:59.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:59 on dist-test-slave-rgbw"
08:03:59.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.902928 2300 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.008s sys 0.004s
08:03:59.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:03:59.905149 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.189:40817: Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:03:59.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.909974 2324 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:03:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.911130 2300 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
08:03:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.911410 2300 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:03:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91200c15660b4e97b54ebd3774fbcc1c"
08:03:59.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:59 on dist-test-slave-rgbw"
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.911706 2300 fs_report.cc:389] FS layout report
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/data
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:03:59.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:03:59.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.967262 2300 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:03:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.974808 2300 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:03:59.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:03:59.975390 2300 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:00.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.080566 2300 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.189:40817
08:04:00.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.080597 2377 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.189:40817 every 8 connection(s)
08:04:00.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.084733 2300 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:04:00.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.093120 2378 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:00.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.093179 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 2300
08:04:00.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.093664 2190 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal/instance
08:04:00.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.104930 2378 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } has no permanent_uuid. Determining permanent_uuid...
08:04:00.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.35.188@KRBTEST.COM; defaulting to no policy
08:04:00.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:00.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.35.188@KRBTEST.COM" created.
08:04:00.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:00 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683439, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.189@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:00.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.147285 2378 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } has no permanent_uuid. Determining permanent_uuid...
08:04:00.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:00.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:00.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:00.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:00 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683439, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.189@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:00.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.35.188@KRBTEST.COM; defaulting to no policy
08:04:00.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:00.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.35.188@KRBTEST.COM" created.
08:04:00.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.178378 2378 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } has no permanent_uuid. Determining permanent_uuid...
08:04:00.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.181443 2326 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:00.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.185967 2378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:00.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:00.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:00.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:00.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.209555 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/logs
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:00.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:00.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:00.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.188:38047
08:04:00.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.188
08:04:00.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:00.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:00.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.188
08:04:00.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:00.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:00.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:00.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:00.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:00.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:00.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:00.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:00.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:00.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:00.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:00.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:00.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:00.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:00.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.235502 2378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 1
08:04:00.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.239754 2378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:00.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.308662 2378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 2
08:04:00.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.313896 2378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:00.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.405412 2378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 3
08:04:00.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.411317 2378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:00.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.554816 2378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 4
08:04:00.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.559943 2378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:00.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.581473 2388 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:00.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.582134 2388 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:00.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.582831 2388 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:00.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.616313 2388 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:00.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.616609 2388 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:00.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.617049 2388 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:00.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.617386 2388 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:00.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.617710 2388 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:00.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.618031 2388 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:00.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.618276 2388 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.671624 2388 master_runner.cc:387] Master server non-default flags:
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.188
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.188:38047
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:00.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.188
08:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/logs
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:00.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.675787 2388 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:00.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.677359 2388 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:00.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.696437 2394 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:00.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:00.859360 2378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 5
08:04:00.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.867774 2378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:01.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:01.415338 2378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 6
08:04:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:01.453233 2378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:01.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:01.995617 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } attempt: 8
08:04:02.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:02 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683437, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:02.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.074239 2291 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } has no permanent_uuid. Determining permanent_uuid...
08:04:02.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.114382 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:02.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.179986 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 1
08:04:02.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.186618 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:02.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.240036 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 2
08:04:02.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.246249 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:02.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.322686 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 3
08:04:02.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.329208 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:02.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:00.698591 2395 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:02.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.371721 2388 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.675s user 0.000s sys 0.002s
08:04:02.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.372130 2388 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.676s user 0.000s sys 0.003s
08:04:02.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.097254 2393 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 2388
08:04:02.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.372249 2396 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1673 milliseconds
08:04:02.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.374046 2397 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:02.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.373976 2388 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:02.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.377629 2388 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:02.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.381009 2388 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:02.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.382481 2388 hybrid_clock.cc:648] HybridClock initialized: now 1781683442382436 us; error 56 us; skew 500 ppm
08:04:02.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.383258 2388 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:02.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:02 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683442, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:02.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.389156 2388 init.cc:377] Logged in from keytab as kudu/127.2.35.188@KRBTEST.COM (short username kudu)
08:04:02.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.395300 2388 webserver.cc:533] Webserver started at http://127.2.35.188:34419/ using document root <none> and password file <none>
08:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.396157 2388 fs_manager.cc:362] Metadata directory not provided
08:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.396341 2388 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:02.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.396704 2388 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:04:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.401069 2388 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/instance:
08:04:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f1831dbc910444c90906336b5d5e1ef"
08:04:02.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:02 on dist-test-slave-rgbw"
08:04:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.402088 2388 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal/instance:
08:04:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f1831dbc910444c90906336b5d5e1ef"
08:04:02.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:02 on dist-test-slave-rgbw"
08:04:02.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.410118 2388 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
08:04:02.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.416520 2412 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.417744 2388 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
08:04:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.418103 2388 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f1831dbc910444c90906336b5d5e1ef"
08:04:02.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:02 on dist-test-slave-rgbw"
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.418511 2388 fs_report.cc:389] FS layout report
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/data
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:02.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:02.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.481016 2388 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:02.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.484648 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 4
08:04:02.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.487996 2388 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:02.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.488511 2388 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:02.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.490106 2291 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:02.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.513883 2378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 7
08:04:02.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.522914 2378 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.2.35.188:38047: Network error: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:02.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.576200 2388 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.188:38047
08:04:02.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.576361 2465 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.188:38047 every 8 connection(s)
08:04:02.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.578933 2388 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:02.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.583112 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 2388
08:04:02.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.583611 2190 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal/instance
08:04:02.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.585963 2466 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
08:04:02.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.602099 2466 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } has no permanent_uuid. Determining permanent_uuid...
08:04:02.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:02 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683434, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:02.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:02 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683442, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.188@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:02.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.638424 2466 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } has no permanent_uuid. Determining permanent_uuid...
08:04:02.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:02 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683434, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:02 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683442, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.188@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:02.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.667568 2466 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } has no permanent_uuid. Determining permanent_uuid...
08:04:02.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:02 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683434, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.35.188@KRBTEST.COM
08:04:02.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:02 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683442, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.188@KRBTEST.COM for kudu/127.2.35.188@KRBTEST.COM
08:04:02.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.706112 2466 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Bootstrap starting.
08:04:02.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.713368 2466 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Neither blocks nor log segments found. Creating new log.
08:04:02.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.714968 2466 log.cc:826] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Log is configured to *not* fsync() on all Append() calls
08:04:02.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.720144 2466 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: No bootstrap required, opened a new log
08:04:02.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.743013 2466 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:02.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.743865 2466 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:02.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.744145 2466 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9f1831dbc910444c90906336b5d5e1ef, State: Initialized, Role: FOLLOWER
08:04:02.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.745254 2466 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:02.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.748374 2478 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:02.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.749220 2478 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: FOLLOWER
08:04:02.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.750044 2466 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: configured and running, proceeding with master startup.
08:04:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.786490 2291 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 5
08:04:02.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.786932 2489 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:04:02.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.787235 2489 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:04:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:02 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683437, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for kudu/127.2.35.188@KRBTEST.COM
08:04:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.779695 2388 master_runner.cc:187] Error getting master registration for 127.2.35.190:32797: OK, instance_id { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" instance_seqno: 1781683437477552 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:04:02.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.821034 2388 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:04:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.825209 2291 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Bootstrap starting.
08:04:02.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.832664 2291 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Neither blocks nor log segments found. Creating new log.
08:04:02.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.834784 2291 log.cc:826] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Log is configured to *not* fsync() on all Append() calls
08:04:02.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.839993 2291 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: No bootstrap required, opened a new log
08:04:02.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.861675 2291 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:02.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.862353 2291 raft_consensus.cc:385] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:02.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.862654 2291 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8fc933c428a74e04993ae685149e88d5, State: Initialized, Role: FOLLOWER
08:04:02.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.863610 2291 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:02.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.865399 2478 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:02.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.866143 2478 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.868193 2492 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.869578 2492 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: This master's current role is: FOLLOWER
08:04:02.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.871097 2266 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8fc933c428a74e04993ae685149e88d5" is_pre_election: true
08:04:02.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.871769 2478 leader_election.cc:290] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797), 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817)
08:04:02.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.870529 2291 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: configured and running, proceeding with master startup.
08:04:02.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.871912 2266 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9f1831dbc910444c90906336b5d5e1ef in term 0.
08:04:02.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.878756 2415 leader_election.cc:304] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8fc933c428a74e04993ae685149e88d5, 9f1831dbc910444c90906336b5d5e1ef; no voters:
08:04:02.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.879690 2478 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 0 FOLLOWER]: Leader pre-election won for term 1
08:04:02.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.880009 2478 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:04:02.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.880298 2478 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 0 FOLLOWER]: Advancing to term 1
08:04:02.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.888785 2478 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:02.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.898587 2266 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8fc933c428a74e04993ae685149e88d5"
08:04:02.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.899233 2266 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 0 FOLLOWER]: Advancing to term 1
08:04:02.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.905793 2478 leader_election.cc:290] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 1 election: Requested vote from peers 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797), 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817)
08:04:02.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.915827 1692 debug-util.cc:398] Leaking SignalData structure 0x7b0800097100 after lost signal to thread 1623
08:04:02.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.916918 1692 debug-util.cc:398] Leaking SignalData structure 0x7b08000761c0 after lost signal to thread 1695
08:04:02.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.919497 2266 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9f1831dbc910444c90906336b5d5e1ef in term 1.
08:04:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.929478 2415 leader_election.cc:304] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8fc933c428a74e04993ae685149e88d5, 9f1831dbc910444c90906336b5d5e1ef; no voters:
08:04:02.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.930393 2478 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Leader election won for term 1
08:04:02.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.931774 2245 tablet.cc:2406] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:04:02.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.947903 2478 raft_consensus.cc:697] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 LEADER]: Becoming Leader. State: Replica: 9f1831dbc910444c90906336b5d5e1ef, State: Running, Role: LEADER
08:04:02.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.949198 2478 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:02.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:02.995276 2418 tablet.cc:2406] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Can't schedule compaction. Clean time has not been advanced past its initial value.
08:04:02.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.997521 2503 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9f1831dbc910444c90906336b5d5e1ef. Latest consensus state: current_term: 1 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:02.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.998302 2503 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: LEADER
08:04:03.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.006100 2507 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
08:04:03.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:03.008951 2504 catalog_manager.cc:1593] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
08:04:03.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:03.009608 2504 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
08:04:03.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.029531 2507 catalog_manager.cc:1519] Initializing Kudu cluster ID...
08:04:03.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:02.973423 2218 master_runner.cc:187] Error getting master registration for 127.2.35.189:40817: OK, instance_id { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" instance_seqno: 1781683440056243 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
08:04:03.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.046690 2218 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:04:03.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.063472 2266 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 1 FOLLOWER]: Refusing update from remote peer 9f1831dbc910444c90906336b5d5e1ef: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:04:03.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.065129 2478 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [LEADER]: Connected to new peer: Peer: permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
08:04:03.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.094868 2492 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9f1831dbc910444c90906336b5d5e1ef. Latest consensus state: current_term: 1 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:03.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.095567 2492 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: This master's current role is: FOLLOWER
08:04:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.114447 2503 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:03.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.115159 2503 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: LEADER
08:04:03.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.117496 2507 catalog_manager.cc:1382] Generated new cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:03.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.117805 2507 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
08:04:03.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.117861 2492 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:03.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.118520 2492 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: This master's current role is: FOLLOWER
08:04:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.145464 2507 catalog_manager.cc:1405] Generated new certificate authority record
08:04:03.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.148931 2507 catalog_manager.cc:1539] Loading token signing keys...
08:04:03.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.191373 2507 catalog_manager.cc:6069] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Generated new TSK 0
08:04:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.196828 2507 catalog_manager.cc:1549] Initializing in-progress tserver states...
08:04:03.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.35.129@KRBTEST.COM; defaulting to no policy
08:04:03.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:03.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.35.129@KRBTEST.COM" created.
08:04:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.129 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:03.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.129 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:03.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.35.129@KRBTEST.COM; defaulting to no policy
08:04:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:03.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.35.129@KRBTEST.COM" created.
08:04:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.129 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.129 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:03.347553 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:03.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/wal
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/data
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/logs
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/data/info.pb
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.129:0
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.35.129
08:04:03.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.129
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.129
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:03.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:04.017277 2504 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:04.017581 2504 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: loading cluster ID for follower catalog manager: success
08:04:04.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:04.031437 2504 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: acquiring CA information for follower catalog manager: success
08:04:04.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:04.043556 2517 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:04.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:04.044188 2517 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:04.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:04.044497 2504 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:04.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:04.045162 2517 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:04.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:04.105949 2517 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:04.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:04.107481 2517 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.35.129
08:04:04.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:04.177031 2517 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:04.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:04.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:04.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:04.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:04.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/data
08:04:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/wal
08:04:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.129
08:04:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.129:0
08:04:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/data/info.pb
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.129
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.35.129
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/logs
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:04.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:04.178802 2517 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:04.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:04.180234 2517 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:04.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:04.222044 2523 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:04.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:04.613327 2378 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } attempt: 8
08:04:04.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:04.622982 2414 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.2.35.189:40817 timed out after 1.746s (SENT)
08:04:04.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:04.678994 2414 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.2.35.189:40817, user_credentials={real_user=slave}} blocked reactor thread for 56062.1us
08:04:04.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:04 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683439, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.189@KRBTEST.COM for kudu/127.2.35.188@KRBTEST.COM
08:04:04.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:04.830564 2378 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Bootstrap starting.
08:04:04.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:04.869400 2378 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Neither blocks nor log segments found. Creating new log.
08:04:04.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:04.933264 2378 log.cc:826] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Log is configured to *not* fsync() on all Append() calls
08:04:05.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.030902 2378 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: No bootstrap required, opened a new log
08:04:05.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.518556 2378 raft_consensus.cc:359] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:05.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.519563 2378 raft_consensus.cc:385] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
08:04:05.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.519857 2378 raft_consensus.cc:740] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 91200c15660b4e97b54ebd3774fbcc1c, State: Initialized, Role: FOLLOWER
08:04:05.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.520887 2378 consensus_queue.cc:260] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.534024 2537 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.535049 2537 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:05.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.546487 2378 sys_catalog.cc:565] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: configured and running, proceeding with master startup.
08:04:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.546628 2349 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "91200c15660b4e97b54ebd3774fbcc1c"
08:04:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.547348 2349 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 0 FOLLOWER]: Advancing to term 1
08:04:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.549788 2350 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "91200c15660b4e97b54ebd3774fbcc1c" is_pre_election: true
08:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.555887 2352 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "91200c15660b4e97b54ebd3774fbcc1c"
08:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:05.557449 2352 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:54496 (request call id 2) took 2650 ms (client timeout 1749 ms). Trace:
08:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:05.557957 2352 rpcz_store.cc:269] 0617 08:04:02.906928 (+ 0us) service_pool.cc:167] Inserting onto call queue
08:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:02.907089 (+ 161us) service_pool.cc:224] Handling call
08:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:05.557326 (+2650237us) inbound_call.cc:177] Queueing success response
08:04:05.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
08:04:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.561096 2353 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "91200c15660b4e97b54ebd3774fbcc1c" is_pre_election: true
08:04:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:05.561954 2353 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:54496 (request call id 1) took 2685 ms (client timeout 1746 ms). Trace:
08:04:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:05.562326 2353 rpcz_store.cc:269] 0617 08:04:02.876325 (+ 0us) service_pool.cc:167] Inserting onto call queue
08:04:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:02.876461 (+ 136us) service_pool.cc:224] Handling call
08:04:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:05.561895 (+2685434us) inbound_call.cc:177] Queueing success response
08:04:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
08:04:05.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.571012 2349 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9f1831dbc910444c90906336b5d5e1ef in term 1.
08:04:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.586211 2351 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 1 FOLLOWER]: Refusing update from remote peer 9f1831dbc910444c90906336b5d5e1ef: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
08:04:05.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.595156 2351 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:54496 (request call id 3) took 2532 ms. Trace:
08:04:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.597023 2351 rpcz_store.cc:276] 0617 08:04:03.062386 (+ 0us) service_pool.cc:167] Inserting onto call queue
08:04:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:03.062510 (+ 124us) service_pool.cc:224] Handling call
08:04:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:05.585568 (+2523058us) raft_consensus.cc:1471] Updating replica for 0 ops
08:04:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:05.586653 (+ 1085us) raft_consensus.cc:1697] Filling consensus response to leader.
08:04:05.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:05.594820 (+ 8167us) spinlock_profiling.cc:243] Waited 21.3 ms on lock 0x7b5400030550. stack: 00007fed9d5f7c83 00007fed9d5f7b80 00007fed9d168574 000000000050eee4 000000000050ee79 00007feda469b951 00007feda4731c8d 00007feda5762b19 00007feda3e14d34 00007feda3e14cc2 00007feda3e14bf1 00007feda3e14b6c 00007feda3e13e22 00007feda2403c3c 00007feda2403076 00007feda2402a1f
08:04:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:05.595052 (+ 232us) inbound_call.cc:177] Queueing success response
08:04:05.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":49061248}
08:04:05.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.600205 2533 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [LEADER]: Connected to new peer: Peer: permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
08:04:05.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.753914 2537 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9f1831dbc910444c90906336b5d5e1ef. Latest consensus state: current_term: 1 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.754793 2537 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:05.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.754315 2550 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:05.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.765828 2550 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.788363 2554 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:05.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.788650 2554 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: loading cluster ID for follower catalog manager: success
08:04:05.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.823141 2554 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: acquiring CA information for follower catalog manager: success
08:04:05.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:05.847093 2554 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:06.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:06.165048 2558 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:06.181560 2558 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: LEADER
08:04:07.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:07 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683447, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:07.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.303493 2235 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:07.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:07.430116 2173 debug-util.cc:398] Leaking SignalData structure 0x7b08000adbc0 after lost signal to thread 2048
08:04:07.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:07.431321 2173 debug-util.cc:398] Leaking SignalData structure 0x7b08000adba0 after lost signal to thread 2176
08:04:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:05.620676 2522 debug-util.cc:398] Leaking SignalData structure 0x7b08000370e0 after lost signal to thread 2517
08:04:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:07.497860 2522 kernel_stack_watchdog.cc:198] Thread 2517 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 390ms:
08:04:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:04:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:04:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:04:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:04:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:04.226724 2524 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:07.499933 2517 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.272s user 0.502s sys 0.927s
08:04:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:07.500494 2517 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.273s user 0.502s sys 0.927s
08:04:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:07.501343 2526 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:07.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:07.508270 2525 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 3279 milliseconds
08:04:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.508265 2517 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:07.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.510366 2517 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:07.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.519441 2517 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:07.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.527336 2517 hybrid_clock.cc:648] HybridClock initialized: now 1781683447527249 us; error 88 us; skew 500 ppm
08:04:07.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.528442 2517 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:07.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:07 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683447, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:07.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.537082 2517 init.cc:377] Logged in from keytab as kudu/127.2.35.129@KRBTEST.COM (short username kudu)
08:04:07.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.572309 2517 webserver.cc:533] Webserver started at http://127.2.35.129:35621/ using document root <none> and password file <none>
08:04:07.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.573887 2517 fs_manager.cc:362] Metadata directory not provided
08:04:07.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.575026 2517 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:07.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.575762 2517 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:04:07.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.582206 2517 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/data/instance:
08:04:07.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "847f21d014eb45918a004c118c3fc21d"
08:04:07.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:07 on dist-test-slave-rgbw"
08:04:07.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.586890 2517 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/wal/instance:
08:04:07.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "847f21d014eb45918a004c118c3fc21d"
08:04:07.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:07 on dist-test-slave-rgbw"
08:04:07.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.617367 2517 fs_manager.cc:696] Time spent creating directory manager: real 0.028s user 0.006s sys 0.004s
08:04:07.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.633155 2565 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.636672 2517 fs_manager.cc:730] Time spent opening block manager: real 0.015s user 0.005s sys 0.001s
08:04:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.637379 2517 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/wal
08:04:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "847f21d014eb45918a004c118c3fc21d"
08:04:07.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:07 on dist-test-slave-rgbw"
08:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.638063 2517 fs_report.cc:389] FS layout report
08:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/wal
08:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/wal
08:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/data/data
08:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:07.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:07.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:07.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.852939 2517 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.875088 2517 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:07.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.875953 2517 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.890692 2517 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:07.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.911435 2517 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:04:07.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.912027 2517 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.000s
08:04:07.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.912514 2517 ts_tablet_manager.cc:616] Registered 0 tablets
08:04:07.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:07.912891 2517 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:04:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.726861 2517 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.129:40017
08:04:08.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.726948 2681 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.129:40017 every 8 connection(s)
08:04:08.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.731058 2517 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/data/info.pb
08:04:08.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.740352 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 2517
08:04:08.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.740926 2190 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-0/wal/instance
08:04:08.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.35.130@KRBTEST.COM; defaulting to no policy
08:04:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:08 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683447, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.129@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:08.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:08.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.35.130@KRBTEST.COM" created.
08:04:08.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:08 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683447, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.129@KRBTEST.COM for kudu/127.2.35.188@KRBTEST.COM
08:04:08.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:08 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683447, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.129@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:08.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.870066 2687 heartbeater.cc:344] Connected to a master server at 127.2.35.190:32797
08:04:08.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.870573 2687 heartbeater.cc:461] Registering TS with master...
08:04:08.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.871869 2687 heartbeater.cc:507] Master 127.2.35.190:32797 requested a full tablet report, sending...
08:04:08.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.876327 2682 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:08.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.876681 2682 heartbeater.cc:461] Registering TS with master...
08:04:08.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.877794 2682 heartbeater.cc:507] Master 127.2.35.189:40817 requested a full tablet report, sending...
08:04:08.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.878978 2256 ts_manager.cc:194] Registered new tserver with Master: 847f21d014eb45918a004c118c3fc21d (127.2.35.129:40017)
08:04:08.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.881479 2343 ts_manager.cc:194] Registered new tserver with Master: 847f21d014eb45918a004c118c3fc21d (127.2.35.129:40017)
08:04:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.891001 2683 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.891404 2683 heartbeater.cc:461] Registering TS with master...
08:04:08.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.892314 2683 heartbeater.cc:507] Master 127.2.35.188:38047 requested a full tablet report, sending...
08:04:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.130 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.130 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.895038 2431 ts_manager.cc:194] Registered new tserver with Master: 847f21d014eb45918a004c118c3fc21d (127.2.35.129:40017)
08:04:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:08.895701 2190 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.2.35.130: real 0.138s user 0.000s sys 0.014s
08:04:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.898000 2431 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.2.35.129@KRBTEST.COM'} at 127.2.35.129:33637
08:04:08.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.35.130@KRBTEST.COM; defaulting to no policy
08:04:08.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:08.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.35.130@KRBTEST.COM" created.
08:04:08.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.130 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.130 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:08.954324 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/wal
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/data
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/logs
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/data/info.pb
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:08.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.130:0
08:04:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.35.130
08:04:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.130
08:04:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.130
08:04:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:09.418576 2696 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:09.419162 2696 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:09.420126 2696 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:09.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:09.456192 2696 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:09.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:09.457285 2696 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.35.130
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:09.510614 2696 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/data
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/wal
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.130
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.130:0
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/data/info.pb
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.130
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:09.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.35.130
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/logs
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:09.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:09.512393 2696 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:09.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:09.514004 2696 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:09.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:09.558656 2702 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:09.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:09 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683449, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:09.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:09.892829 2318 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:09.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:09.904544 2683 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:11.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:10.947479 2701 debug-util.cc:398] Leaking SignalData structure 0x7b08000370e0 after lost signal to thread 2696
08:04:11.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:09.569434 2703 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:11.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:11.747390 2701 kernel_stack_watchdog.cc:198] Thread 2696 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 378ms:
08:04:11.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:04:11.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:04:11.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:04:11.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:04:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:11.750167 2705 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:11.749423 2696 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.182s user 0.671s sys 1.298s
08:04:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:11.751106 2696 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.184s user 0.672s sys 1.298s
08:04:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:11.754815 2704 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2184 milliseconds
08:04:11.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.755292 2696 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.756670 2696 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:11.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.759435 2696 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:11.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.760862 2696 hybrid_clock.cc:648] HybridClock initialized: now 1781683451760902 us; error 116 us; skew 500 ppm
08:04:11.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.761921 2696 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:11.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:11 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683451, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:11.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.770049 2696 init.cc:377] Logged in from keytab as kudu/127.2.35.130@KRBTEST.COM (short username kudu)
08:04:11.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.779253 2696 webserver.cc:533] Webserver started at http://127.2.35.130:42825/ using document root <none> and password file <none>
08:04:11.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.780169 2696 fs_manager.cc:362] Metadata directory not provided
08:04:11.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.780407 2696 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:11.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.780969 2696 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:04:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.785333 2696 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/data/instance:
08:04:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a39b773fc5ee472ca6ae106471a57777"
08:04:11.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:11 on dist-test-slave-rgbw"
08:04:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.786415 2696 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/wal/instance:
08:04:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a39b773fc5ee472ca6ae106471a57777"
08:04:11.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:11 on dist-test-slave-rgbw"
08:04:11.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.793373 2696 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.002s
08:04:11.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.798661 2716 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:11.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.799665 2696 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.003s
08:04:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.800028 2696 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/wal
08:04:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a39b773fc5ee472ca6ae106471a57777"
08:04:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:11 on dist-test-slave-rgbw"
08:04:11.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.800352 2696 fs_report.cc:389] FS layout report
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/wal
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/wal
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/data/data
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:11.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:11.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.855772 2696 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:11.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.861304 2696 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:11.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.861801 2696 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.864940 2696 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:11.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.869029 2696 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:04:11.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.869249 2696 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:04:11.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.869572 2696 ts_tablet_manager.cc:616] Registered 0 tablets
08:04:11.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:11.869771 2696 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:04:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.039230 2696 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.130:34455
08:04:12.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.039335 2828 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.130:34455 every 8 connection(s)
08:04:12.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.041878 2696 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/data/info.pb
08:04:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.052065 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 2696
08:04:12.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.053001 2190 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-1/wal/instance
08:04:12.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:12 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683451, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.130@KRBTEST.COM for kudu/127.2.35.188@KRBTEST.COM
08:04:12.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:12 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683451, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.130@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:12.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.2.35.131@KRBTEST.COM; defaulting to no policy
08:04:12.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.2.35.131@KRBTEST.COM" created.
08:04:12.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.109536 2830 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:12.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.110087 2830 heartbeater.cc:461] Registering TS with master...
08:04:12.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.111454 2830 heartbeater.cc:507] Master 127.2.35.188:38047 requested a full tablet report, sending...
08:04:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.114785 2431 ts_manager.cc:194] Registered new tserver with Master: a39b773fc5ee472ca6ae106471a57777 (127.2.35.130:34455)
08:04:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.116518 2431 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.2.35.130@KRBTEST.COM'} at 127.2.35.130:42399
08:04:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.123907 2829 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:12 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683451, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.130@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.124253 2829 heartbeater.cc:461] Registering TS with master...
08:04:12.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.125319 2829 heartbeater.cc:507] Master 127.2.35.189:40817 requested a full tablet report, sending...
08:04:12.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.127962 2343 ts_manager.cc:194] Registered new tserver with Master: a39b773fc5ee472ca6ae106471a57777 (127.2.35.130:34455)
08:04:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.131 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:12.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.2.35.131 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.139771 2831 heartbeater.cc:344] Connected to a master server at 127.2.35.190:32797
08:04:12.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.140154 2831 heartbeater.cc:461] Registering TS with master...
08:04:12.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.140852 2831 heartbeater.cc:507] Master 127.2.35.190:32797 requested a full tablet report, sending...
08:04:12.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.143664 2256 ts_manager.cc:194] Registered new tserver with Master: a39b773fc5ee472ca6ae106471a57777 (127.2.35.130:34455)
08:04:12.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.2.35.131@KRBTEST.COM; defaulting to no policy
08:04:12.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:12.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.2.35.131@KRBTEST.COM" created.
08:04:12.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
08:04:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.131 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.2.35.131 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab.
08:04:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.182562 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/wal
08:04:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/data
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/logs
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/data/info.pb
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.131:0
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.35.131
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.131
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.131
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:12.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:12.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:12 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683452, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:12.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.394927 2408 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:12.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:12.504683 2844 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:12.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:12.505189 2844 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:12.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:12.505689 2844 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:12.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:12.541407 2844 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:12.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:12.542409 2844 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.2.35.131
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.581606 2844 tablet_server_runner.cc:78] Tablet server non-default flags:
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/data
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/wal
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.131
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.131:0
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/data/info.pb
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.131
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.2.35.131
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/logs
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.582837 2844 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:12.583756 2844 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:12.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:12.597407 2851 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:13.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:13.120543 2830 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:14.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:14.000941 2850 debug-util.cc:398] Leaking SignalData structure 0x7b08000370e0 after lost signal to thread 2844
08:04:14.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:12.598649 2852 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:14.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:14.286832 2844 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.689s user 0.531s sys 1.055s
08:04:14.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:14.287218 2844 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.690s user 0.531s sys 1.055s
08:04:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:14.289710 2854 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:14.291576 2853 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1689 milliseconds
08:04:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.291643 2844 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:14.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.292691 2844 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:14.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.294999 2844 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.296531 2844 hybrid_clock.cc:648] HybridClock initialized: now 1781683454296466 us; error 67 us; skew 500 ppm
08:04:14.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.297365 2844 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:14.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:14 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683454, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:14.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.303736 2844 init.cc:377] Logged in from keytab as kudu/127.2.35.131@KRBTEST.COM (short username kudu)
08:04:14.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.309226 2844 webserver.cc:533] Webserver started at http://127.2.35.131:32867/ using document root <none> and password file <none>
08:04:14.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.310130 2844 fs_manager.cc:362] Metadata directory not provided
08:04:14.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.310313 2844 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:14.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.310684 2844 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
08:04:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.315222 2844 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/data/instance:
08:04:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72b21a1ecbe04e098e2bb3817726f1e1"
08:04:14.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:14 on dist-test-slave-rgbw"
08:04:14.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.316242 2844 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/wal/instance:
08:04:14.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72b21a1ecbe04e098e2bb3817726f1e1"
08:04:14.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:14 on dist-test-slave-rgbw"
08:04:14.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.323210 2844 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.005s
08:04:14.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.329171 2869 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:14.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.331423 2844 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
08:04:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.331816 2844 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/wal
08:04:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "72b21a1ecbe04e098e2bb3817726f1e1"
08:04:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:14 on dist-test-slave-rgbw"
08:04:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.332243 2844 fs_report.cc:389] FS layout report
08:04:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/wal
08:04:14.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/wal
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/data/data
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:14.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.384572 2844 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:14.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.395431 2844 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:14.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.395896 2844 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:14.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.398938 2844 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
08:04:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.403594 2844 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
08:04:14.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.403839 2844 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
08:04:14.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.404541 2844 ts_tablet_manager.cc:616] Registered 0 tablets
08:04:14.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.404733 2844 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
08:04:14.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.571835 2844 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.131:37093
08:04:14.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.571969 2982 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.131:37093 every 8 connection(s)
08:04:14.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.574460 2844 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/data/info.pb
08:04:14.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.585294 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 2844
08:04:14.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.586938 2190 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/ts-2/wal/instance
08:04:14.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:14 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683454, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.131@KRBTEST.COM for kudu/127.2.35.188@KRBTEST.COM
08:04:14.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:14 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683454, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.131@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:14.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:14 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683454, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.131@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.686579 2985 heartbeater.cc:344] Connected to a master server at 127.2.35.190:32797
08:04:14.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.687260 2985 heartbeater.cc:461] Registering TS with master...
08:04:14.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.689782 2985 heartbeater.cc:507] Master 127.2.35.190:32797 requested a full tablet report, sending...
08:04:14.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.695577 2256 ts_manager.cc:194] Registered new tserver with Master: 72b21a1ecbe04e098e2bb3817726f1e1 (127.2.35.131:37093)
08:04:14.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.708869 2984 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:14.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.709340 2984 heartbeater.cc:461] Registering TS with master...
08:04:14.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.711406 2984 heartbeater.cc:507] Master 127.2.35.188:38047 requested a full tablet report, sending...
08:04:14.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.715543 2430 ts_manager.cc:194] Registered new tserver with Master: 72b21a1ecbe04e098e2bb3817726f1e1 (127.2.35.131:37093)
08:04:14.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.718713 2430 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.2.35.131@KRBTEST.COM'} at 127.2.35.131:48593
08:04:14.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.727567 2983 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:14.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.728163 2983 heartbeater.cc:461] Registering TS with master...
08:04:14.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.729308 2983 heartbeater.cc:507] Master 127.2.35.189:40817 requested a full tablet report, sending...
08:04:14.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.733683 2343 ts_manager.cc:194] Registered new tserver with Master: 72b21a1ecbe04e098e2bb3817726f1e1 (127.2.35.131:37093)
08:04:14.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.751981 2190 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
08:04:14.776 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:14.777 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
08:04:14.787 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
08:04:14.788 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
08:04:14.792 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.2.35.190"
port: 32797
}
bound_http_address {
host: "127.2.35.190"
port: 35165
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.2.35.189"
port: 40817
}
bound_http_address {
host: "127.2.35.189"
port: 45919
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.2.35.188"
port: 38047
}
bound_http_address {
host: "127.2.35.188"
port: 34419
}
}
}
08:04:14.792 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
08:04:14.794 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.2.35.129"
port: 40017
}
bound_http_address {
host: "127.2.35.129"
port: 35621
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.2.35.130"
port: 34455
}
bound_http_address {
host: "127.2.35.130"
port: 42825
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.2.35.131"
port: 37093
}
bound_http_address {
host: "127.2.35.131"
port: 32867
}
}
}
08:04:14.794 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:04:14.795 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:04:14.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:14 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683454, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:14.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:04:14.835 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:14.837 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc
08:04:14.848 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:04:14.849 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc
08:04:14.851 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:04:14.878 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
08:04:14.880 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:04:14.881 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:04:14.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:14 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683454, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:14.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:04:14.914 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:14.914 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc
08:04:14.916 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:04:14.916 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.35.188:38047
08:04:14.918 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:04:14.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.919039 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 2388
08:04:14.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:14.954358 2568 connection.cc:573] client connection to 127.2.35.188:38047 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:04:14.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:14.956357 2683 heartbeater.cc:646] Failed to heartbeat to 127.2.35.188:38047 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
08:04:14.960 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:14.960 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.35.189:40817
08:04:14.961 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:04:14.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.961469 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 2300
08:04:14.995 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:14.996 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.35.190:32797
08:04:14.996 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:04:14.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:14.996641 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 2218
08:04:15.029 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:15.029 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.35.188:38047
08:04:15.033 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:04:15.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:15.034830 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:15.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:15.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/logs
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.188:38047
08:04:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.188
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34419
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.188
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:15.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.134393 2830 heartbeater.cc:646] Failed to heartbeat to 127.2.35.188:38047 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.2.35.188:38047: connect: Connection refused (error 111)
08:04:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.364811 2997 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.365481 2997 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.366313 2997 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:15.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.401002 2997 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:15.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.401299 2997 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:15.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.401731 2997 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:15.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.402060 2997 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:15.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.402431 2997 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:15.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.402774 2997 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:15.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.403024 2997 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:04:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:15.442533 2997 master_runner.cc:387] Master server non-default flags:
08:04:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data
08:04:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:15.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.188
08:04:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.188:38047
08:04:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:15.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.188
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34419
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/logs
08:04:15.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:15.444264 2997 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:15.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:15.445741 2997 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:15.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.458109 3005 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:15.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.720345 2985 heartbeater.cc:646] Failed to heartbeat to 127.2.35.190:32797 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:15.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:15.724674 2984 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:15.458590 3006 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:17.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:16.860312 3004 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 2997
08:04:17.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.289785 2997 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.832s user 0.643s sys 1.129s
08:04:17.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.290833 2997 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.833s user 0.644s sys 1.129s
08:04:17.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.292151 3008 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:17.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.297314 3007 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1837 milliseconds
08:04:17.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.297340 2997 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:17.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.298596 2997 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:17.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.300933 2997 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:17.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.302268 2997 hybrid_clock.cc:648] HybridClock initialized: now 1781683457302241 us; error 49 us; skew 500 ppm
08:04:17.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.303018 2997 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:17.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:17 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683457, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:17.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.309377 2997 init.cc:377] Logged in from keytab as kudu/127.2.35.188@KRBTEST.COM (short username kudu)
08:04:17.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.314085 2997 webserver.cc:533] Webserver started at http://127.2.35.188:34419/ using document root <none> and password file <none>
08:04:17.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.314978 2997 fs_manager.cc:362] Metadata directory not provided
08:04:17.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.315230 2997 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:17.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.323350 2997 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.001s
08:04:17.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.327772 3028 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:17.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.328868 2997 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:04:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.329214 2997 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f1831dbc910444c90906336b5d5e1ef"
08:04:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:02 on dist-test-slave-rgbw"
08:04:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.331118 2997 fs_report.cc:389] FS layout report
08:04:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/data
08:04:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:17.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:17.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:17.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.385015 2997 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:17.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.390693 2997 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:17.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.391225 2997 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.462878 2997 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.188:38047
08:04:17.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.462960 3079 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.188:38047 every 8 connection(s)
08:04:17.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.465685 2997 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:17.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.469034 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 2997
08:04:17.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:17.470 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.35.189:40817
08:04:17.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:04:17.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.472460 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/logs
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.189:40817
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.189
08:04:17.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45919
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.189
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:17.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:17.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.480000 3080 sys_catalog.cc:263] Verifying existing consensus state
08:04:17.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.485277 3080 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Bootstrap starting.
08:04:17.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.527175 3080 log.cc:826] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Log is configured to *not* fsync() on all Append() calls
08:04:17.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.543298 3080 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:17.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.544006 3080 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Bootstrap complete.
08:04:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:17 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683457, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:17.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.551124 2561 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:17.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.568619 3080 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:17.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.572115 3080 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9f1831dbc910444c90906336b5d5e1ef, State: Initialized, Role: FOLLOWER
08:04:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.573304 3080 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.577298 3084 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:17.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.578361 3084 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: FOLLOWER
08:04:17.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.579118 3080 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: configured and running, proceeding with master startup.
08:04:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.619190 3095 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:17.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.619449 3095 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: loading cluster ID for follower catalog manager: success
08:04:17.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.626842 3095 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: acquiring CA information for follower catalog manager: success
08:04:17.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.633199 3095 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:17.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.615926 2997 master_runner.cc:187] Error getting master registration for 127.2.35.190:32797: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111),
08:04:17.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.640358 2997 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:04:17.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.811369 2984 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:17.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.811887 2984 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:17.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.818079 3045 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "72b21a1ecbe04e098e2bb3817726f1e1" instance_seqno: 1781683454539342) as {username='kudu', principal='kudu/127.2.35.131@KRBTEST.COM'} at 127.2.35.131:37663; Asking this server to re-register.
08:04:17.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.819865 2984 heartbeater.cc:461] Registering TS with master...
08:04:17.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.823098 3045 ts_manager.cc:194] Registered new tserver with Master: 72b21a1ecbe04e098e2bb3817726f1e1 (127.2.35.131:37093)
08:04:17.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.872006 3082 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:17.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.872637 3082 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:17.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.873297 3082 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:17.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.906627 3082 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.906909 3082 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.907369 3082 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:17.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.907696 3082 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:17.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.908042 3082 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:17.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.908375 3082 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:17.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.908661 3082 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:04:17.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.950172 3082 master_runner.cc:387] Master server non-default flags:
08:04:17.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:17.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:17.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:17.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.189
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.189:40817
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.189
08:04:17.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45919
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/logs
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:17.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.952421 3082 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:17.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:17.953850 3082 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:17.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.966104 3107 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:18.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:18.061173 2683 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:18.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:18.065928 3045 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "847f21d014eb45918a004c118c3fc21d" instance_seqno: 1781683448629114) as {username='kudu', principal='kudu/127.2.35.129@KRBTEST.COM'} at 127.2.35.129:50939; Asking this server to re-register.
08:04:18.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:18.073077 2683 heartbeater.cc:461] Registering TS with master...
08:04:18.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:18.076722 3045 ts_manager.cc:194] Registered new tserver with Master: 847f21d014eb45918a004c118c3fc21d (127.2.35.129:40017)
08:04:18.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:18.210897 2830 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:18.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:18.214545 3045 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a39b773fc5ee472ca6ae106471a57777" instance_seqno: 1781683452002663) as {username='kudu', principal='kudu/127.2.35.130@KRBTEST.COM'} at 127.2.35.130:58125; Asking this server to re-register.
08:04:18.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:18.217096 2830 heartbeater.cc:461] Registering TS with master...
08:04:18.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:18.221143 3045 ts_manager.cc:194] Registered new tserver with Master: a39b773fc5ee472ca6ae106471a57777 (127.2.35.130:34455)
08:04:19.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.302225 3125 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:19.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.303635 3125 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:19.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.329008 3125 leader_election.cc:290] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797), 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817)
08:04:19.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:19.329672 3030 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:04:19.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:19.344046 3030 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817): Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:04:19.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:19.344880 3031 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:19.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.345824 3031 leader_election.cc:304] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9f1831dbc910444c90906336b5d5e1ef; no voters: 8fc933c428a74e04993ae685149e88d5, 91200c15660b4e97b54ebd3774fbcc1c
08:04:19.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.347172 3125 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
08:04:19.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:19.368571 3106 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 3082
08:04:19.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:19.717758 3082 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.749s user 0.355s sys 0.871s
08:04:19.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:19.719866 3082 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.752s user 0.355s sys 0.871s
08:04:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:19.720948 3109 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1751 milliseconds
08:04:19.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:19.721411 3110 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:19.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.721804 3082 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:19.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:17.968143 3108 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:19.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.727504 3082 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:19.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.733575 3082 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:19.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.735016 3082 hybrid_clock.cc:648] HybridClock initialized: now 1781683459734955 us; error 64 us; skew 500 ppm
08:04:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.736161 3082 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:19.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:19 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683459, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:19.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.743582 3082 init.cc:377] Logged in from keytab as kudu/127.2.35.189@KRBTEST.COM (short username kudu)
08:04:19.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.749975 3082 webserver.cc:533] Webserver started at http://127.2.35.189:45919/ using document root <none> and password file <none>
08:04:19.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.751065 3082 fs_manager.cc:362] Metadata directory not provided
08:04:19.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.751315 3082 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:19.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.760672 3082 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
08:04:19.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.765957 3135 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:19.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.767164 3082 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
08:04:19.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.767534 3082 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:19.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91200c15660b4e97b54ebd3774fbcc1c"
08:04:19.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:59 on dist-test-slave-rgbw"
08:04:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.769574 3082 fs_report.cc:389] FS layout report
08:04:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/data
08:04:19.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:19.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:19.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.825755 3082 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:19.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.836289 3082 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:19.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.836699 3082 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:19.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.923606 3082 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.189:40817
08:04:19.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.923669 3187 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.189:40817 every 8 connection(s)
08:04:19.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.926488 3082 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:04:19.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.935084 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 3082
08:04:19.939 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:19.939 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.35.190:32797
08:04:19.940 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:04:19.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.942592 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/logs
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:19.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:19.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:19.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:19.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:19.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:19.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:19.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.190:32797
08:04:19.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.190
08:04:19.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35165
08:04:19.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:19.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.190
08:04:19.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:19.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:19.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:19.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.966012 3188 sys_catalog.cc:263] Verifying existing consensus state
08:04:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:19.975207 3188 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Bootstrap starting.
08:04:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.026058 3188 log.cc:826] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Log is configured to *not* fsync() on all Append() calls
08:04:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.043649 3188 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:20.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.044651 3188 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Bootstrap complete.
08:04:20.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.075793 3188 raft_consensus.cc:359] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:20.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.079774 3188 raft_consensus.cc:740] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 91200c15660b4e97b54ebd3774fbcc1c, State: Initialized, Role: FOLLOWER
08:04:20.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.080973 3188 consensus_queue.cc:260] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:20.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.127135 3196 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.129575 3196 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:20.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.129164 3188 sys_catalog.cc:565] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: configured and running, proceeding with master startup.
08:04:20.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:20 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683457, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.129@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:20.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.159765 2682 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:20.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.188827 3208 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:20.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.192847 3208 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: loading cluster ID for follower catalog manager: success
08:04:20.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.182859 3082 master_runner.cc:187] Error getting master registration for 127.2.35.190:32797: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111),
08:04:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.209460 3208 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: acquiring CA information for follower catalog manager: success
08:04:20.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.209872 3082 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:04:20.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.216089 3208 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:20.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.246690 2829 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:20.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.253567 3153 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a39b773fc5ee472ca6ae106471a57777" instance_seqno: 1781683452002663) as {username='kudu', principal='kudu/127.2.35.130@KRBTEST.COM'} at 127.2.35.130:42615; Asking this server to re-register.
08:04:20.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.255143 2829 heartbeater.cc:461] Registering TS with master...
08:04:20.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.259228 3152 ts_manager.cc:194] Registered new tserver with Master: a39b773fc5ee472ca6ae106471a57777 (127.2.35.130:34455)
08:04:20.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.429945 3190 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:20.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.430611 3190 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:20.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.431581 3190 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:20.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.478286 3190 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:20.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.478626 3190 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:20.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.479108 3190 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:20.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.479507 3190 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.479861 3190 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.480199 3190 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.480476 3190 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.530522 3190 master_runner.cc:387] Master server non-default flags:
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.190
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.190:32797
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:20.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.190
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35165
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/logs
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:20.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:20.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:20.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:20.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:20.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:20.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.532202 3190 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:20.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.533589 3190 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.551403 3217 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:20.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.855719 2983 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:20.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.859772 3153 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "72b21a1ecbe04e098e2bb3817726f1e1" instance_seqno: 1781683454539342) as {username='kudu', principal='kudu/127.2.35.131@KRBTEST.COM'} at 127.2.35.131:49149; Asking this server to re-register.
08:04:20.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.862237 2983 heartbeater.cc:461] Registering TS with master...
08:04:20.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:20.867151 3153 ts_manager.cc:194] Registered new tserver with Master: 72b21a1ecbe04e098e2bb3817726f1e1 (127.2.35.131:37093)
08:04:21.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.053087 3224 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:21.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.055783 3224 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:21.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.094034 3224 leader_election.cc:290] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797), 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817)
08:04:21.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:21.133517 3031 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:21.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.153541 3163 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "91200c15660b4e97b54ebd3774fbcc1c" is_pre_election: true
08:04:21.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.154896 3163 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9f1831dbc910444c90906336b5d5e1ef in term 1.
08:04:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.157562 3030 leader_election.cc:304] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 91200c15660b4e97b54ebd3774fbcc1c, 9f1831dbc910444c90906336b5d5e1ef; no voters: 8fc933c428a74e04993ae685149e88d5
08:04:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.159286 3224 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Leader pre-election won for term 2
08:04:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.159747 3224 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:04:21.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.160172 3224 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 1 FOLLOWER]: Advancing to term 2
08:04:21.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.166455 3153 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "847f21d014eb45918a004c118c3fc21d" instance_seqno: 1781683448629114) as {username='kudu', principal='kudu/127.2.35.129@KRBTEST.COM'} at 127.2.35.129:33019; Asking this server to re-register.
08:04:21.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.168442 2682 heartbeater.cc:461] Registering TS with master...
08:04:21.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.175280 3153 ts_manager.cc:194] Registered new tserver with Master: 847f21d014eb45918a004c118c3fc21d (127.2.35.129:40017)
08:04:21.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.199162 3224 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:21.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:21.220079 3031 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:21.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.223625 3163 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "91200c15660b4e97b54ebd3774fbcc1c"
08:04:21.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.224459 3163 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 1 FOLLOWER]: Advancing to term 2
08:04:21.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.239436 3224 leader_election.cc:290] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 2 election: Requested vote from peers 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797), 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817)
08:04:21.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.246690 3163 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9f1831dbc910444c90906336b5d5e1ef in term 2.
08:04:21.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.247961 3030 leader_election.cc:304] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 91200c15660b4e97b54ebd3774fbcc1c, 9f1831dbc910444c90906336b5d5e1ef; no voters: 8fc933c428a74e04993ae685149e88d5
08:04:21.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.249507 3224 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Leader election won for term 2
08:04:21.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.257539 3224 raft_consensus.cc:697] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 LEADER]: Becoming Leader. State: Replica: 9f1831dbc910444c90906336b5d5e1ef, State: Running, Role: LEADER
08:04:21.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.259044 3224 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:21.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.279482 3230 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9f1831dbc910444c90906336b5d5e1ef. Latest consensus state: current_term: 2 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:21.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.280932 3230 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: LEADER
08:04:21.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.290694 3232 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
08:04:21.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.301429 3232 catalog_manager.cc:1519] Initializing Kudu cluster ID...
08:04:21.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.306799 3232 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:21.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.307381 3232 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
08:04:21.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.313284 3232 catalog_manager.cc:1539] Loading token signing keys...
08:04:21.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.318233 3232 catalog_manager.cc:6080] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Loaded TSK: 0
08:04:21.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.319972 3232 catalog_manager.cc:1549] Initializing in-progress tserver states...
08:04:21.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:21 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683461, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:21.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.775955 2712 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:21.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.853524 3163 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 2 FOLLOWER]: Refusing update from remote peer 9f1831dbc910444c90906336b5d5e1ef: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
08:04:21.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.855369 3234 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [LEADER]: Connected to new peer: Peer: permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
08:04:21.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:21.877547 3031 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef -> Peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Couldn't send request to peer 8fc933c428a74e04993ae685149e88d5. Status: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:21.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.910557 3238 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9f1831dbc910444c90906336b5d5e1ef. Latest consensus state: current_term: 2 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:21.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.912895 3233 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:21.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.915006 3233 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: LEADER
08:04:21.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.921947 3238 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:21.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.929577 3238 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:21.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:21.930920 3238 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:21.952554 3216 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 3190
08:04:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:22.592065 3216 kernel_stack_watchdog.cc:198] Thread 3190 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
08:04:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:04:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:04:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:04:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:04:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:20.554936 3218 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:22.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:22.592886 3190 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.040s user 0.741s sys 1.052s
08:04:22.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:22.593452 3190 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.040s user 0.741s sys 1.052s
08:04:22.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.604029 3190 server_base.cc:1061] running on GCE node
08:04:22.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:22.605012 3250 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.606631 3190 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:22.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.609503 3190 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:22.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.610911 3190 hybrid_clock.cc:648] HybridClock initialized: now 1781683462610754 us; error 153 us; skew 500 ppm
08:04:22.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.611967 3190 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:22.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:22 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683462, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:22.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.618602 3190 init.cc:377] Logged in from keytab as kudu/127.2.35.190@KRBTEST.COM (short username kudu)
08:04:22.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.624403 3190 webserver.cc:533] Webserver started at http://127.2.35.190:35165/ using document root <none> and password file <none>
08:04:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.625368 3190 fs_manager.cc:362] Metadata directory not provided
08:04:22.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.625581 3190 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:22.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.633610 3190 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.003s sys 0.004s
08:04:22.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.638746 3256 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:22.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.640069 3190 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
08:04:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.640441 3190 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fc933c428a74e04993ae685149e88d5"
08:04:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:57 on dist-test-slave-rgbw"
08:04:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.642855 3190 fs_report.cc:389] FS layout report
08:04:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/data
08:04:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:22.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:22.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:22.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:22.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:22.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:22.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:22.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:22.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:22.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.643297 2997 master_runner.cc:187] Error getting master registration for 127.2.35.190:32797: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111),
08:04:22.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.643707 2997 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:04:22.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.714262 3190 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:22.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.720434 3190 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:22.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.720963 3190 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:22.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.826678 3190 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.190:32797
08:04:22.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.826560 3308 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.190:32797 every 8 connection(s)
08:04:22.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.830593 3190 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:04:22.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.834769 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 3190
08:04:22.836 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:22.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.849006 3309 sys_catalog.cc:263] Verifying existing consensus state
08:04:22.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.855338 3309 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Bootstrap starting.
08:04:22.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.859242 2984 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:22.922 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.35.188:38047(127.2.35.188:38047)] connecting to peer
08:04:22.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.928637 3309 log.cc:826] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Log is configured to *not* fsync() on all Append() calls
08:04:22.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.957863 3309 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:22.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.958875 3309 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Bootstrap complete.
08:04:22.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:22.980258 2985 heartbeater.cc:344] Connected to a master server at 127.2.35.190:32797
08:04:23.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:22 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683457, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.188@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:23.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.023104 3309 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.027355 3309 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8fc933c428a74e04993ae685149e88d5, State: Initialized, Role: FOLLOWER
08:04:23.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.028467 3309 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:23.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.065783 3316 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:23.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.067116 3316 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: This master's current role is: FOLLOWER
08:04:23.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.068177 3309 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: configured and running, proceeding with master startup.
08:04:23.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.068853 3284 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 1 FOLLOWER]: Advancing to term 2
08:04:23.073 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.35.189:40817(127.2.35.189:40817)] connecting to peer
08:04:23.085 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.35.190:32797(127.2.35.190:32797)] connecting to peer
08:04:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.118408 3316 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9f1831dbc910444c90906336b5d5e1ef. Latest consensus state: current_term: 2 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.119248 3316 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: This master's current role is: FOLLOWER
08:04:23.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.122342 3323 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:23.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.123095 3323 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: This master's current role is: FOLLOWER
08:04:23.124 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.2.35.189:40817(127.2.35.189:40817)] Successfully connected to peer
08:04:23.130 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.2.35.188:38047(127.2.35.188:38047)] Successfully connected to peer
08:04:23.132 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.2.35.190:32797(127.2.35.190:32797)] Successfully connected to peer
08:04:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.134497 2683 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:23.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:23 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683457, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.129@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:23.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.212926 2687 heartbeater.cc:344] Connected to a master server at 127.2.35.190:32797
08:04:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.249159 2830 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.249579 3336 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:04:23.264 [WARN - kudu-nio-2] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
08:04:23.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:23 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683461, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.130@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:23.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:23 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683462, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:23.308 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.35.188
08:04:23.313 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.35.189
08:04:23.321 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.35.190
08:04:23.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.338625 2831 heartbeater.cc:344] Connected to a master server at 127.2.35.190:32797
08:04:23.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.345014 3274 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a39b773fc5ee472ca6ae106471a57777" instance_seqno: 1781683452002663) as {username='kudu', principal='kudu/127.2.35.130@KRBTEST.COM'} at 127.2.35.130:50017; Asking this server to re-register.
08:04:23.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.346737 2831 heartbeater.cc:461] Registering TS with master...
08:04:23.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.350672 3273 ts_manager.cc:194] Registered new tserver with Master: a39b773fc5ee472ca6ae106471a57777 (127.2.35.130:34455)
08:04:23.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:23 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683462, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:23.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.383153 3345 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:23.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.386245 3345 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: LEADER
08:04:23.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:23 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683462, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for kudu/127.2.35.188@KRBTEST.COM
08:04:23.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:23 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781683454, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.35.188@KRBTEST.COM
08:04:23.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:23 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781683454, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:23.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:23 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781683454, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:23.773 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x8e472a93, L:/127.0.0.1:35954 - R:/127.2.35.188:38047] using SASL/GSSAPI
08:04:23.773 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0xc42fce73, L:/127.0.0.1:34890 - R:/127.2.35.190:32797] using SASL/GSSAPI
08:04:23.773 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xaebd94f9, L:/127.0.0.1:58876 - R:/127.2.35.189:40817] using SASL/GSSAPI
08:04:23.824 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
08:04:23.825 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=985), Traces: [0ms] refreshing cache from master, [33ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.35.188:38047, [229ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.35.189:40817, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.2.35.190:32797, [952ms] Sub RPC ConnectToMaster: received response from server master-127.2.35.189:40817: OK, [952ms] Sub RPC ConnectToMaster: received response from server master-127.2.35.190:32797: OK, [953ms] Sub RPC ConnectToMaster: received response from server master-127.2.35.188:38047: OK, deferred=null) after lookup
08:04:23.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.986500 3273 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "72b21a1ecbe04e098e2bb3817726f1e1" instance_seqno: 1781683454539342) as {username='kudu', principal='kudu/127.2.35.131@KRBTEST.COM'} at 127.2.35.131:50137; Asking this server to re-register.
08:04:23.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.987797 2985 heartbeater.cc:461] Registering TS with master...
08:04:23.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:23.990175 3273 ts_manager.cc:194] Registered new tserver with Master: 72b21a1ecbe04e098e2bb3817726f1e1 (127.2.35.131:37093)
08:04:24.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:24.218344 3273 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "847f21d014eb45918a004c118c3fc21d" instance_seqno: 1781683448629114) as {username='kudu', principal='kudu/127.2.35.129@KRBTEST.COM'} at 127.2.35.129:45427; Asking this server to re-register.
08:04:24.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:24.219669 2687 heartbeater.cc:461] Registering TS with master...
08:04:24.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:24.222296 3273 ts_manager.cc:194] Registered new tserver with Master: 847f21d014eb45918a004c118c3fc21d (127.2.35.129:40017)
08:04:24.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:24.255956 3336 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:24.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:24.256237 3336 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: loading cluster ID for follower catalog manager: success
08:04:24.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:24.261899 3336 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: acquiring CA information for follower catalog manager: success
08:04:24.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:24.266315 3336 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:24.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:24 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683464, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:24.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:24.308136 2865 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:24.860 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:04:24.860 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:04:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:24 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683464, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:24.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:04:24.901 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:24.902 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc
08:04:24.903 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:04:24.904 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.35.188:38047
08:04:24.904 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:04:24.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:24.904788 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 2997
08:04:24.946 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.2.35.188:38047(127.2.35.188:38047)] handling channelInactive
08:04:24.947 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.2.35.188:38047(127.2.35.188:38047)] cleaning up while in state READY due to: connection closed
08:04:24.954 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:24.954 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.35.189:40817
08:04:24.954 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:04:24.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:24.955260 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 3082
08:04:24.995 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.2.35.189:40817(127.2.35.189:40817)] handling channelInactive
08:04:24.995 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.2.35.189:40817(127.2.35.189:40817)] cleaning up while in state READY due to: connection closed
08:04:24.996 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:24.996 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.35.190:32797
08:04:24.997 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:04:24.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:24.997383 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 3190
08:04:25.027 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:25.027 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.2.35.190:32797(127.2.35.190:32797)] handling channelInactive
08:04:25.027 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.35.188:38047
08:04:25.027 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.2.35.190:32797(127.2.35.190:32797)] cleaning up while in state READY due to: connection closed
08:04:25.029 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:25.031064 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/logs
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:25.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.188:38047
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.188
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34419
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.188
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:25.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:25.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:25.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.348716 3352 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:25.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.349411 3352 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:25.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.350481 3352 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:25.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.388906 3352 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:25.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.389138 3352 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:25.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.389492 3352 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:25.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.389705 3352 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.389899 3352 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.390110 3352 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:25.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.390257 3352 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:04:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:25.429708 3352 master_runner.cc:387] Master server non-default flags:
08:04:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data
08:04:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:25.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.188
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.188:38047
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.188
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34419
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/logs
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:25.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:25.431020 3352 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:25.431939 3352 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:25.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.443701 3361 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:25.444893 3362 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:26.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:26.772527 3363 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
08:04:26.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:26.774204 3364 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:26.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:26.776943 3352 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.333s user 0.531s sys 0.792s
08:04:26.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:26.777210 3352 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.333s user 0.531s sys 0.792s
08:04:26.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.777506 3352 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:26.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.778669 3352 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:26.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.785514 3352 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:26.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.787063 3352 hybrid_clock.cc:648] HybridClock initialized: now 1781683466786930 us; error 129 us; skew 500 ppm
08:04:26.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.787999 3352 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:26.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:26 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683466, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:26.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.796095 3352 init.cc:377] Logged in from keytab as kudu/127.2.35.188@KRBTEST.COM (short username kudu)
08:04:26.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.802184 3352 webserver.cc:533] Webserver started at http://127.2.35.188:34419/ using document root <none> and password file <none>
08:04:26.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.803433 3352 fs_manager.cc:362] Metadata directory not provided
08:04:26.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.803712 3352 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:26.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.817009 3352 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.006s sys 0.000s
08:04:26.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.822223 3375 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:26.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.823542 3352 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.823917 3352 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f1831dbc910444c90906336b5d5e1ef"
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:02 on dist-test-slave-rgbw"
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.826076 3352 fs_report.cc:389] FS layout report
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/data
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:26.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:26.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:26.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.922320 3352 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.932425 3352 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:26.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:26.933015 3352 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:27.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.017262 3352 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.188:38047
08:04:27.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.017408 3427 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.188:38047 every 8 connection(s)
08:04:27.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.020634 3352 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:27.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.029385 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 3352
08:04:27.030 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:27.030 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.35.189:40817
08:04:27.030 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:04:27.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.031599 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/logs
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.189:40817
08:04:27.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.189
08:04:27.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45919
08:04:27.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:27.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.189
08:04:27.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:27.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:27.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:27.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:27.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.031988 3428 sys_catalog.cc:263] Verifying existing consensus state
08:04:27.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.040508 3428 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Bootstrap starting.
08:04:27.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.093518 3428 log.cc:826] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Log is configured to *not* fsync() on all Append() calls
08:04:27.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.111397 3428 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:27.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.112250 3428 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Bootstrap complete.
08:04:27.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.135834 3428 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:27.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.138160 3428 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9f1831dbc910444c90906336b5d5e1ef, State: Initialized, Role: FOLLOWER
08:04:27.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.139235 3428 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:27.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.142040 3432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:27.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.142992 3432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: FOLLOWER
08:04:27.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.143898 3428 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: configured and running, proceeding with master startup.
08:04:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.177030 3439 negotiation.cc:336] Failed RPC negotiation. Trace:
08:04:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:27.153939 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.2.35.129:45305 (local address 127.2.35.188:38047)
08:04:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:27.166430 (+ 12491us) server_negotiation.cc:207] Beginning negotiation
08:04:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:27.166477 (+ 47us) server_negotiation.cc:400] Waiting for connection header
08:04:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:27.168826 (+ 2349us) server_negotiation.cc:408] Connection header received
08:04:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:27.170092 (+ 1266us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:04:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:27.170154 (+ 62us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:04:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:27.171085 (+ 931us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:04:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:27.171618 (+ 533us) server_negotiation.cc:227] Negotiated authn=SASL
08:04:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:27.174832 (+ 3214us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.2.35.129:45305: BlockingRecv error: recv got EOF from 127.2.35.129:45305 (error 108)
08:04:27.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":11690,"thread_start_us":11168,"threads_started":1}
08:04:27.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.191567 3445 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:27.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.191833 3445 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: loading cluster ID for follower catalog manager: success
08:04:27.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.199059 3445 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: acquiring CA information for follower catalog manager: success
08:04:27.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.205642 3445 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:27.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.185043 3352 master_runner.cc:187] Error getting master registration for 127.2.35.190:32797: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111),
08:04:27.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.220460 3352 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:04:27.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.283216 2830 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.289474 3393 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a39b773fc5ee472ca6ae106471a57777" instance_seqno: 1781683452002663) as {username='kudu', principal='kudu/127.2.35.130@KRBTEST.COM'} at 127.2.35.130:41737; Asking this server to re-register.
08:04:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.290946 2830 heartbeater.cc:461] Registering TS with master...
08:04:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.294426 3393 ts_manager.cc:194] Registered new tserver with Master: a39b773fc5ee472ca6ae106471a57777 (127.2.35.130:34455)
08:04:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.446722 3430 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:27.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.447371 3430 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:27.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.448242 3430 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:27.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.482733 3430 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:27.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.483016 3430 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:27.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.483423 3430 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:27.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.483757 3430 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.484124 3430 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:27.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.484458 3430 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:27.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.484702 3430 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:04:27.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.525482 3430 master_runner.cc:387] Master server non-default flags:
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.189
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.189:40817
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:27.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:04:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.189
08:04:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45919
08:04:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/logs
08:04:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:27.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.526885 3430 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:27.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.528224 3430 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:27.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.541755 3454 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:27.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:27 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683467, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:27.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.554508 2561 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:27.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.912357 2984 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:27.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.916465 3393 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "72b21a1ecbe04e098e2bb3817726f1e1" instance_seqno: 1781683454539342) as {username='kudu', principal='kudu/127.2.35.131@KRBTEST.COM'} at 127.2.35.131:55805; Asking this server to re-register.
08:04:27.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.918049 2984 heartbeater.cc:461] Registering TS with master...
08:04:27.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:27.921247 3393 ts_manager.cc:194] Registered new tserver with Master: 72b21a1ecbe04e098e2bb3817726f1e1 (127.2.35.131:37093)
08:04:28.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:28.189191 2683 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:28.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:28.195431 3393 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "847f21d014eb45918a004c118c3fc21d" instance_seqno: 1781683448629114) as {username='kudu', principal='kudu/127.2.35.129@KRBTEST.COM'} at 127.2.35.129:44821; Asking this server to re-register.
08:04:28.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:28.197139 2683 heartbeater.cc:461] Registering TS with master...
08:04:28.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:28.200073 3393 ts_manager.cc:194] Registered new tserver with Master: 847f21d014eb45918a004c118c3fc21d (127.2.35.129:40017)
08:04:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:28.405529 1889 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7360 after lost signal to thread 1763
08:04:28.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:28.407300 1889 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad0a0 after lost signal to thread 1892
08:04:28.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:28.613415 3465 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:28.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:28.629884 3465 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:28.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:28.674919 3379 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:28.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:28.688310 3379 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:28.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:28.694510 3465 leader_election.cc:290] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797), 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817)
08:04:28.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:28.699096 3378 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817): Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:04:28.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:28.700937 3378 leader_election.cc:304] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9f1831dbc910444c90906336b5d5e1ef; no voters: 8fc933c428a74e04993ae685149e88d5, 91200c15660b4e97b54ebd3774fbcc1c
08:04:28.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:28.709795 3468 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
08:04:29.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:28.942504 3453 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 3430
08:04:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:29.336792 3453 kernel_stack_watchdog.cc:198] Thread 3430 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
08:04:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:04:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:04:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:04:29.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:04:29.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:27.543304 3455 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:29.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:29.354351 3430 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.812s user 0.413s sys 0.906s
08:04:29.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:29.355111 3430 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.813s user 0.413s sys 0.906s
08:04:29.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:29.365665 3474 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:29.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.366648 3430 server_base.cc:1061] running on GCE node
08:04:29.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.368857 3430 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:29.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.373445 3430 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:29.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.375139 3430 hybrid_clock.cc:648] HybridClock initialized: now 1781683469375060 us; error 70 us; skew 500 ppm
08:04:29.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.376683 3430 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:29.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:29 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683469, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:29.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.385430 3430 init.cc:377] Logged in from keytab as kudu/127.2.35.189@KRBTEST.COM (short username kudu)
08:04:29.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.391759 3430 webserver.cc:533] Webserver started at http://127.2.35.189:45919/ using document root <none> and password file <none>
08:04:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.393066 3430 fs_manager.cc:362] Metadata directory not provided
08:04:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.393381 3430 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:29.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.405572 3430 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.003s sys 0.004s
08:04:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.412178 3480 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.413537 3430 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.002s
08:04:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.413928 3430 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91200c15660b4e97b54ebd3774fbcc1c"
08:04:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:59 on dist-test-slave-rgbw"
08:04:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.416160 3430 fs_report.cc:389] FS layout report
08:04:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:29.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/data
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:29.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.491448 3430 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:29.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.498835 3430 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:29.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.499383 3430 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.636657 3430 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.189:40817
08:04:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.636739 3532 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.189:40817 every 8 connection(s)
08:04:29.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.639984 3430 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:04:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.642817 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 3430
08:04:29.644 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:29.644 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.35.190:32797
08:04:29.644 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:04:29.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.647680 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:29.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:29.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:29.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data
08:04:29.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:29.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:29.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:29.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/logs
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.190:32797
08:04:29.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.190
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35165
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.190
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:29.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:29.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.687534 3533 sys_catalog.cc:263] Verifying existing consensus state
08:04:29.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.696314 3533 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Bootstrap starting.
08:04:29.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.760529 3533 log.cc:826] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Log is configured to *not* fsync() on all Append() calls
08:04:29.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.830055 3533 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:29.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.831087 3533 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Bootstrap complete.
08:04:29.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.879200 3533 raft_consensus.cc:359] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:29.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.882769 3533 raft_consensus.cc:740] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 91200c15660b4e97b54ebd3774fbcc1c, State: Initialized, Role: FOLLOWER
08:04:29.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.884316 3533 consensus_queue.cc:260] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:29.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.897949 3537 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:29.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.898801 3537 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:29.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:29.899855 3533 sys_catalog.cc:565] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: configured and running, proceeding with master startup.
08:04:29.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:29 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683464, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.131@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:30.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.000809 3550 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
08:04:30.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.027166 2983 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:30.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.038591 3498 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "72b21a1ecbe04e098e2bb3817726f1e1" instance_seqno: 1781683454539342) as {username='kudu', principal='kudu/127.2.35.131@KRBTEST.COM'} at 127.2.35.131:55495; Asking this server to re-register.
08:04:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.045737 2983 heartbeater.cc:461] Registering TS with master...
08:04:30.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.049453 3498 ts_manager.cc:194] Registered new tserver with Master: 72b21a1ecbe04e098e2bb3817726f1e1 (127.2.35.131:37093)
08:04:30.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.057624 3430 master_runner.cc:187] Error getting master registration for 127.2.35.190:32797: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111),
08:04:30.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.111732 3430 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:04:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:30 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683467, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.129@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:30.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.298530 2682 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:30.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.301134 3498 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "847f21d014eb45918a004c118c3fc21d" instance_seqno: 1781683448629114) as {username='kudu', principal='kudu/127.2.35.129@KRBTEST.COM'} at 127.2.35.129:38957; Asking this server to re-register.
08:04:30.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.302858 2682 heartbeater.cc:461] Registering TS with master...
08:04:30.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.306090 3498 ts_manager.cc:194] Registered new tserver with Master: 847f21d014eb45918a004c118c3fc21d (127.2.35.129:40017)
08:04:30.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:30 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683461, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.130@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:30.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.397886 2829 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.400811 3535 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.401180 3498 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a39b773fc5ee472ca6ae106471a57777" instance_seqno: 1781683452002663) as {username='kudu', principal='kudu/127.2.35.130@KRBTEST.COM'} at 127.2.35.130:50765; Asking this server to re-register.
08:04:30.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.401623 3535 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:30.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.402729 3535 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:30.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.403403 2829 heartbeater.cc:461] Registering TS with master...
08:04:30.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.407011 3498 ts_manager.cc:194] Registered new tserver with Master: a39b773fc5ee472ca6ae106471a57777 (127.2.35.130:34455)
08:04:30.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.472847 3535 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:30.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.473510 3535 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:30.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.474231 3535 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:30.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.474635 3535 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:30.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.475026 3535 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:30.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.475405 3535 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:30.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.475703 3535 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:04:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.508886 3563 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:30.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.509668 3563 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:30.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.511608 3563 leader_election.cc:290] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797), 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817)
08:04:30.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.532485 3379 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:30.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:30 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683466, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.188@KRBTEST.COM for kudu/127.2.35.189@KRBTEST.COM
08:04:30.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.548559 3508 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "91200c15660b4e97b54ebd3774fbcc1c" is_pre_election: true
08:04:30.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.549364 3508 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9f1831dbc910444c90906336b5d5e1ef in term 2.
08:04:30.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.550479 3378 leader_election.cc:304] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 91200c15660b4e97b54ebd3774fbcc1c, 9f1831dbc910444c90906336b5d5e1ef; no voters: 8fc933c428a74e04993ae685149e88d5
08:04:30.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.551530 3563 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Leader pre-election won for term 3
08:04:30.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.551801 3563 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:04:30.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.552120 3563 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 2 FOLLOWER]: Advancing to term 3
08:04:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.558851 3563 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:30.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.560897 3563 leader_election.cc:290] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 3 election: Requested vote from peers 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797), 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817)
08:04:30.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.561507 3508 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "91200c15660b4e97b54ebd3774fbcc1c"
08:04:30.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.561985 3508 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 2 FOLLOWER]: Advancing to term 3
08:04:30.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.562404 3535 master_runner.cc:387] Master server non-default flags:
08:04:30.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.190
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.190:32797
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.190
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35165
08:04:30.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/logs
08:04:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:30.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:30.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.564281 3535 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:30.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.565711 3535 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:30.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.571130 3508 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9f1831dbc910444c90906336b5d5e1ef in term 3.
08:04:30.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.572173 3378 leader_election.cc:304] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 91200c15660b4e97b54ebd3774fbcc1c, 9f1831dbc910444c90906336b5d5e1ef; no voters:
08:04:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.572965 3563 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Leader election won for term 3
08:04:30.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.574363 3379 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:30.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.576201 3563 raft_consensus.cc:697] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 LEADER]: Becoming Leader. State: Replica: 9f1831dbc910444c90906336b5d5e1ef, State: Running, Role: LEADER
08:04:30.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.577380 3563 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:30.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.581573 3569 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:30.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.591943 3567 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9f1831dbc910444c90906336b5d5e1ef. Latest consensus state: current_term: 3 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:30.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.592561 3567 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: LEADER
08:04:30.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.607504 3573 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
08:04:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.620175 3573 catalog_manager.cc:1519] Initializing Kudu cluster ID...
08:04:30.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.626869 3573 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:30.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.627578 3573 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
08:04:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.636175 3573 catalog_manager.cc:1539] Loading token signing keys...
08:04:30.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.642089 3573 catalog_manager.cc:6080] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Loaded TSK: 0
08:04:30.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.644109 3573 catalog_manager.cc:1549] Initializing in-progress tserver states...
08:04:30.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.959656 3508 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 3 FOLLOWER]: Refusing update from remote peer 9f1831dbc910444c90906336b5d5e1ef: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
08:04:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:30.962108 3567 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [LEADER]: Connected to new peer: Peer: permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
08:04:31.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.015448 3575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9f1831dbc910444c90906336b5d5e1ef. Latest consensus state: current_term: 3 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:31.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.018577 3575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:31.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.030033 3575 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:31.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.031921 3575 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:31.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.040933 3563 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.041592 3563 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: LEADER
08:04:31.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:31.043547 3379 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef -> Peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Couldn't send request to peer 8fc933c428a74e04993ae685149e88d5. Status: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:31.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.059311 3550 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:31.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.059649 3550 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: loading cluster ID for follower catalog manager: success
08:04:31.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.067206 3550 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: acquiring CA information for follower catalog manager: success
08:04:31.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.073182 3550 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:31.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:31 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683471, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:31.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.781242 2712 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:31.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:31.978431 2984 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.223667 3352 master_runner.cc:187] Error getting master registration for 127.2.35.190:32797: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111),
08:04:32.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.224118 3352 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:04:32.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.277313 2683 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:32.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.362967 2830 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:32.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:31.983258 3568 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 3535
08:04:32.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:32.417423 3535 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.835s user 0.670s sys 1.058s
08:04:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:32.417773 3535 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.835s user 0.670s sys 1.058s
08:04:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:32.416553 3568 kernel_stack_watchdog.cc:198] Thread 3535 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
08:04:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
08:04:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
08:04:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
08:04:32.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
08:04:32.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:30.585187 3570 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:32.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:32.429505 3589 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:32.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.430071 3535 server_base.cc:1061] running on GCE node
08:04:32.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.431520 3535 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.434707 3535 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:32.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.436237 3535 hybrid_clock.cc:648] HybridClock initialized: now 1781683472436174 us; error 48 us; skew 500 ppm
08:04:32.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.437304 3535 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:32.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:32 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683472, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:32.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.444084 3535 init.cc:377] Logged in from keytab as kudu/127.2.35.190@KRBTEST.COM (short username kudu)
08:04:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.449579 3535 webserver.cc:533] Webserver started at http://127.2.35.190:35165/ using document root <none> and password file <none>
08:04:32.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.450802 3535 fs_manager.cc:362] Metadata directory not provided
08:04:32.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.451074 3535 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:32.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.459733 3535 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
08:04:32.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.464504 3596 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:32.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.465665 3535 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.466044 3535 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fc933c428a74e04993ae685149e88d5"
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:57 on dist-test-slave-rgbw"
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.467962 3535 fs_report.cc:389] FS layout report
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/data
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:32.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:32.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:32.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:32.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:32.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:32.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:32.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:32.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.526484 3535 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:32.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.536482 3535 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:32.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.537016 3535 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:32.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.646463 3535 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.190:32797
08:04:32.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.652426 3535 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:04:32.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.646523 3648 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.190:32797 every 8 connection(s)
08:04:32.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.662000 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 3535
08:04:32.663 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:32.664 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.35.188:38047(127.2.35.188:38047)] connecting to peer
08:04:32.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.684546 3649 sys_catalog.cc:263] Verifying existing consensus state
08:04:32.689 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.2.35.188:38047(127.2.35.188:38047)] Successfully connected to peer
08:04:32.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.698796 3649 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Bootstrap starting.
08:04:32.712 [DEBUG - kudu-nio-3] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.2.35.188
08:04:32.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.758085 3649 log.cc:826] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Log is configured to *not* fsync() on all Append() calls
08:04:32.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.802757 3649 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:32.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.803821 3649 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Bootstrap complete.
08:04:32.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:32 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1781683464, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.2.35.188@KRBTEST.COM
08:04:32.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.848135 3649 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:32.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.851934 3649 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8fc933c428a74e04993ae685149e88d5, State: Initialized, Role: FOLLOWER
08:04:32.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.853260 3649 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.860415 3656 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:32.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.861263 3656 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: This master's current role is: FOLLOWER
08:04:32.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.862216 3649 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: configured and running, proceeding with master startup.
08:04:32.882 [DEBUG - kudu-nio-3] (Negotiator.java:919) Authenticated connection [id: 0x4b356cbc, L:/127.0.0.1:45140 - R:/127.2.35.188:38047] using SASL/GSSAPI
08:04:32.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.921502 3667 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:32.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.921778 3667 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: loading cluster ID for follower catalog manager: success
08:04:32.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.929569 3667 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: acquiring CA information for follower catalog manager: success
08:04:32.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:32.946460 3667 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:32.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:32 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683472, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:33.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.169787 3623 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 2 FOLLOWER]: Advancing to term 3
08:04:33.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.175256 2985 heartbeater.cc:344] Connected to a master server at 127.2.35.190:32797
08:04:33.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.189934 3677 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:33.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.191395 3677 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: This master's current role is: FOLLOWER
08:04:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.197300 3656 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9f1831dbc910444c90906336b5d5e1ef. Latest consensus state: current_term: 3 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:33.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.197994 3656 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [sys.catalog]: This master's current role is: FOLLOWER
08:04:33.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.200129 3613 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "72b21a1ecbe04e098e2bb3817726f1e1" instance_seqno: 1781683454539342) as {username='kudu', principal='kudu/127.2.35.131@KRBTEST.COM'} at 127.2.35.131:43141; Asking this server to re-register.
08:04:33.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.201730 2985 heartbeater.cc:461] Registering TS with master...
08:04:33.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.204955 3613 ts_manager.cc:194] Registered new tserver with Master: 72b21a1ecbe04e098e2bb3817726f1e1 (127.2.35.131:37093)
08:04:33.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.332665 2687 heartbeater.cc:344] Connected to a master server at 127.2.35.190:32797
08:04:33.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.335464 3613 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "847f21d014eb45918a004c118c3fc21d" instance_seqno: 1781683448629114) as {username='kudu', principal='kudu/127.2.35.129@KRBTEST.COM'} at 127.2.35.129:53317; Asking this server to re-register.
08:04:33.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.336843 2687 heartbeater.cc:461] Registering TS with master...
08:04:33.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.339634 3613 ts_manager.cc:194] Registered new tserver with Master: 847f21d014eb45918a004c118c3fc21d (127.2.35.129:40017)
08:04:33.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.451069 2831 heartbeater.cc:344] Connected to a master server at 127.2.35.190:32797
08:04:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.453774 3613 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a39b773fc5ee472ca6ae106471a57777" instance_seqno: 1781683452002663) as {username='kudu', principal='kudu/127.2.35.130@KRBTEST.COM'} at 127.2.35.130:36919; Asking this server to re-register.
08:04:33.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.455585 2831 heartbeater.cc:461] Registering TS with master...
08:04:33.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.458530 3613 ts_manager.cc:194] Registered new tserver with Master: a39b773fc5ee472ca6ae106471a57777 (127.2.35.130:34455)
08:04:33.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.754676 3567 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:33.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.756232 3567 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: LEADER
08:04:33.887 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
08:04:33.888 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
08:04:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:33 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683473, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:33.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
08:04:33.934 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:33.934 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc
08:04:33.936 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
08:04:33.936 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.35.188:38047
08:04:33.936 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
08:04:33.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.939476 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 3352
08:04:33.987 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.2.35.188:38047(127.2.35.188:38047)] handling channelInactive
08:04:33.987 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:33.987 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.2.35.188:38047(127.2.35.188:38047)] cleaning up while in state READY due to: connection closed
08:04:33.987 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.35.189:40817
08:04:33.987 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
08:04:33.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:33.989059 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 3430
08:04:34.030 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:34.031 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.2.35.190:32797
08:04:34.031 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
08:04:34.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:34.031704 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 3535
08:04:34.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:34.059 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.35.188:38047
08:04:34.059 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
08:04:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:34.061359 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/logs
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.188:38047
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.188
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34419
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.188
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:34.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:34.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:34.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:34 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683474, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:34.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:34.311416 2865 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:34.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.412999 3686 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:34.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.413530 3686 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:34.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.414202 3686 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:34.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.448175 3686 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:34.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.448405 3686 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:34.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.448669 3686 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.448947 3686 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.449167 3686 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.449359 3686 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:34.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.449510 3686 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:04:34.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:34.489312 3686 master_runner.cc:387] Master server non-default flags:
08:04:34.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.188
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.188:38047
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:34.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.188
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34419
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/logs
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:34.490607 3686 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:34.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:34.491683 3686 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:34.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.503032 3695 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:36.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:35.906563 3694 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 3686
08:04:36.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:34.503501 3696 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:36.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.142422 3686 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.639s user 0.563s sys 1.025s
08:04:36.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.142871 3686 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.640s user 0.563s sys 1.025s
08:04:36.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.145015 3698 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:36.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.147806 3697 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1643 milliseconds
08:04:36.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.147830 3686 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.149149 3686 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.151703 3686 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:36.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.153093 3686 hybrid_clock.cc:648] HybridClock initialized: now 1781683476153056 us; error 36 us; skew 500 ppm
08:04:36.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.153997 3686 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:36.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:36 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683476, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:36.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.160709 3686 init.cc:377] Logged in from keytab as kudu/127.2.35.188@KRBTEST.COM (short username kudu)
08:04:36.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.165448 3686 webserver.cc:533] Webserver started at http://127.2.35.188:34419/ using document root <none> and password file <none>
08:04:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.166404 3686 fs_manager.cc:362] Metadata directory not provided
08:04:36.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.166654 3686 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:36.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.174661 3686 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.003s sys 0.002s
08:04:36.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.179250 3711 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:36.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.180459 3686 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
08:04:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.180933 3686 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f1831dbc910444c90906336b5d5e1ef"
08:04:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:04:02 on dist-test-slave-rgbw"
08:04:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.183087 3686 fs_report.cc:389] FS layout report
08:04:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/wal
08:04:36.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/data
08:04:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:36.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:36.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.242925 3686 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:36.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.248710 3686 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:36.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.249200 3686 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:36.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.323835 3686 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.188:38047
08:04:36.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.323964 3763 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.188:38047 every 8 connection(s)
08:04:36.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.326689 3686 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-2/data/info.pb
08:04:36.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.335179 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 3686
08:04:36.335 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:36.336 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.35.189:40817
08:04:36.336 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.337849 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/logs
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:36.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.189:40817
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.189
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45919
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.189
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:36.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.339421 3764 sys_catalog.cc:263] Verifying existing consensus state
08:04:36.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.345288 3764 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Bootstrap starting.
08:04:36.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.385695 3764 log.cc:826] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Log is configured to *not* fsync() on all Append() calls
08:04:36.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.409214 3764 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:36.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.410375 3764 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Bootstrap complete.
08:04:36.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:36 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683471, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.130@KRBTEST.COM for kudu/127.2.35.188@KRBTEST.COM
08:04:36.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.444561 3764 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:36.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.448043 3764 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9f1831dbc910444c90906336b5d5e1ef, State: Initialized, Role: FOLLOWER
08:04:36.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.449146 3764 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.453126 3771 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.454030 3771 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: FOLLOWER
08:04:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.454229 3764 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: configured and running, proceeding with master startup.
08:04:36.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.462785 2830 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:36.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.488940 3782 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:36.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.489228 3782 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: loading cluster ID for follower catalog manager: success
08:04:36.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.496625 3782 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: acquiring CA information for follower catalog manager: success
08:04:36.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.501971 3782 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:36.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.497756 3686 master_runner.cc:187] Error getting master registration for 127.2.35.190:32797: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111),
08:04:36.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.526714 3686 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:04:36.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.714125 3766 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:36.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.714664 3766 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:36.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.715270 3766 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:36.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.769446 3766 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:36.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.769773 3766 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:36.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.770259 3766 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:36.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.770601 3766 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:36.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.770924 3766 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:36.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.771260 3766 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:36.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.771499 3766 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:04:36.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.816437 3766 master_runner.cc:387] Master server non-default flags:
08:04:36.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:36.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.189
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.189:40817
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.189
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45919
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:36.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/logs
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.817883 3766 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:36.818852 3766 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:36.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.830758 3790 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:37.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.110473 2984 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:37.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.116458 3728 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "72b21a1ecbe04e098e2bb3817726f1e1" instance_seqno: 1781683454539342) as {username='kudu', principal='kudu/127.2.35.131@KRBTEST.COM'} at 127.2.35.131:53003; Asking this server to re-register.
08:04:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.118263 2984 heartbeater.cc:461] Registering TS with master...
08:04:37.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.121770 3728 ts_manager.cc:194] Registered new tserver with Master: 72b21a1ecbe04e098e2bb3817726f1e1 (127.2.35.131:37093)
08:04:37.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.367549 2683 heartbeater.cc:344] Connected to a master server at 127.2.35.188:38047
08:04:37.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.371457 3728 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "847f21d014eb45918a004c118c3fc21d" instance_seqno: 1781683448629114) as {username='kudu', principal='kudu/127.2.35.129@KRBTEST.COM'} at 127.2.35.129:48433; Asking this server to re-register.
08:04:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.373061 2683 heartbeater.cc:461] Registering TS with master...
08:04:37.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.376941 3728 ts_manager.cc:194] Registered new tserver with Master: 847f21d014eb45918a004c118c3fc21d (127.2.35.129:40017)
08:04:37.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.470965 3728 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a39b773fc5ee472ca6ae106471a57777" instance_seqno: 1781683452002663) as {username='kudu', principal='kudu/127.2.35.130@KRBTEST.COM'} at 127.2.35.130:43519; Asking this server to re-register.
08:04:37.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.474447 2830 heartbeater.cc:461] Registering TS with master...
08:04:37.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.477977 3728 ts_manager.cc:194] Registered new tserver with Master: a39b773fc5ee472ca6ae106471a57777 (127.2.35.130:34455)
08:04:37.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:37 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683477, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:37.559142 2561 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:38.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:38.153152 3805 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:38.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:38.153714 3805 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:38.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:38.183454 3714 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:38.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:38.184244 3805 leader_election.cc:290] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797), 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817)
08:04:38.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:38.195621 3713 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817): Network error: Client connection negotiation failed: client connection to 127.2.35.189:40817: connect: Connection refused (error 111)
08:04:38.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:38.196398 3714 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:38.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:38.196897 3714 leader_election.cc:304] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 9f1831dbc910444c90906336b5d5e1ef; no voters: 8fc933c428a74e04993ae685149e88d5, 91200c15660b4e97b54ebd3774fbcc1c
08:04:38.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:38.197851 3805 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
08:04:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:38.235577 3789 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 3766
08:04:39.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:39.488333 3792 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2655 milliseconds
08:04:39.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:39.488700 3766 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.656s user 0.003s sys 0.000s
08:04:39.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:39.490640 3766 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.658s user 0.004s sys 0.000s
08:04:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:36.831916 3791 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.497213 3766 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:39.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:39.497800 3794 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:39.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.499332 3766 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:39.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.502054 3766 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:39.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.503623 3766 hybrid_clock.cc:648] HybridClock initialized: now 1781683479503557 us; error 58 us; skew 500 ppm
08:04:39.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.504550 3766 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:39.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:39 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683479, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:39.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.511281 3766 init.cc:377] Logged in from keytab as kudu/127.2.35.189@KRBTEST.COM (short username kudu)
08:04:39.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.517405 3766 webserver.cc:533] Webserver started at http://127.2.35.189:45919/ using document root <none> and password file <none>
08:04:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.518334 3766 fs_manager.cc:362] Metadata directory not provided
08:04:39.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.518590 3766 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.530748 3766 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.005s sys 0.003s
08:04:39.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.535955 3824 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:39.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.537142 3766 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.001s
08:04:39.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.537461 3766 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:39.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91200c15660b4e97b54ebd3774fbcc1c"
08:04:39.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:59 on dist-test-slave-rgbw"
08:04:39.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.539565 3766 fs_report.cc:389] FS layout report
08:04:39.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:39.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/wal
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/data
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:39.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:39.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.610140 3766 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:39.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.615942 3766 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:39.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.616470 3766 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:39.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.733793 3766 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.189:40817
08:04:39.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.733846 3876 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.189:40817 every 8 connection(s)
08:04:39.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.739432 3766 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-1/data/info.pb
08:04:39.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.743673 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 3766
08:04:39.744 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:39.744 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.2.35.190:32797
08:04:39.745 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
08:04:39.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.747449 2190 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/logs
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.190:32797
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.190
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35165
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.190
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
08:04:39.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
08:04:39.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:39.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:39.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:39.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:39.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:39.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:39.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:39.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:39.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
08:04:39.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
08:04:39.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.775579 3877 sys_catalog.cc:263] Verifying existing consensus state
08:04:39.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.786880 3877 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Bootstrap starting.
08:04:39.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.842437 3877 log.cc:826] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Log is configured to *not* fsync() on all Append() calls
08:04:39.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.867308 3877 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:39.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.868427 3877 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: Bootstrap complete.
08:04:39.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.896473 3877 raft_consensus.cc:359] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:39.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.898839 3877 raft_consensus.cc:740] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 91200c15660b4e97b54ebd3774fbcc1c, State: Initialized, Role: FOLLOWER
08:04:39.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.899644 3877 consensus_queue.cc:260] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:39.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.902515 3881 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:39.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.903321 3881 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:39.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.904420 3877 sys_catalog.cc:565] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: configured and running, proceeding with master startup.
08:04:39.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.945508 3892 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:39.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.945791 3892 catalog_manager.cc:1587] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: loading cluster ID for follower catalog manager: success
08:04:39.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.958621 3892 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: acquiring CA information for follower catalog manager: success
08:04:39.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.942481 3766 master_runner.cc:187] Error getting master registration for 127.2.35.190:32797: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111),
08:04:39.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.970597 3766 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:04:39.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:39.973133 3892 catalog_manager.cc:1637] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
08:04:40.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.137364 3878 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
08:04:40.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.137990 3878 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
08:04:40.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.138687 3878 flags.cc:432] Enabled unsafe flag: --never_fsync=true
08:04:40.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.180313 2983 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:40.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.186581 3842 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "72b21a1ecbe04e098e2bb3817726f1e1" instance_seqno: 1781683454539342) as {username='kudu', principal='kudu/127.2.35.131@KRBTEST.COM'} at 127.2.35.131:50935; Asking this server to re-register.
08:04:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.188345 2983 heartbeater.cc:461] Registering TS with master...
08:04:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.188467 3878 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
08:04:40.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.188772 3878 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
08:04:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.189257 3878 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
08:04:40.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.189612 3878 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
08:04:40.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.189976 3878 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
08:04:40.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.190389 3878 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
08:04:40.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.190678 3878 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:39839/jwks.json
08:04:40.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.191918 3842 ts_manager.cc:194] Registered new tserver with Master: 72b21a1ecbe04e098e2bb3817726f1e1 (127.2.35.131:37093)
08:04:40.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.239398 3878 master_runner.cc:387] Master server non-default flags:
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.2.35.148:34179
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.2.35.190:32797,127.2.35.189:40817,127.2.35.188:38047
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/krb5kdc/kudu.keytab
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.2.35.190
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.2.35.190:32797
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:39839/jwks.json
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.2.35.190
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35165
08:04:40.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/logs
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 23b10557f461bc2be3d2c282748d8de96f9c46ec
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 17 Jun 2026 07:43:25 UTC on e7f111948823
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 12435
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
08:04:40.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.240854 3878 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
08:04:40.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.241940 3878 file_cache.cc:492] Constructed file cache file cache with capacity 419430
08:04:40.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.253610 3904 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:40.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.353493 3910 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
08:04:40.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.354537 3910 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:40.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.363967 3910 leader_election.cc:290] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797), 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817)
08:04:40.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.384874 3852 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "91200c15660b4e97b54ebd3774fbcc1c" is_pre_election: true
08:04:40.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.386499 3852 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 9f1831dbc910444c90906336b5d5e1ef in term 3.
08:04:40.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.389266 3713 leader_election.cc:304] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 91200c15660b4e97b54ebd3774fbcc1c, 9f1831dbc910444c90906336b5d5e1ef; no voters:
08:04:40.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.390151 3910 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Leader pre-election won for term 4
08:04:40.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.390588 3910 raft_consensus.cc:493] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
08:04:40.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.390992 3910 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 3 FOLLOWER]: Advancing to term 4
08:04:40.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.401459 3714 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:40.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.404982 3910 raft_consensus.cc:515] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:40.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.415113 3852 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "9f1831dbc910444c90906336b5d5e1ef" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "91200c15660b4e97b54ebd3774fbcc1c"
08:04:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.416059 3852 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 3 FOLLOWER]: Advancing to term 4
08:04:40.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.416874 3910 leader_election.cc:290] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 4 election: Requested vote from peers 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797), 91200c15660b4e97b54ebd3774fbcc1c (127.2.35.189:40817)
08:04:40.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.422493 3714 leader_election.cc:336] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111)
08:04:40.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.437770 2682 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:40.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.447296 3852 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 9f1831dbc910444c90906336b5d5e1ef in term 4.
08:04:40.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.456418 3842 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "847f21d014eb45918a004c118c3fc21d" instance_seqno: 1781683448629114) as {username='kudu', principal='kudu/127.2.35.129@KRBTEST.COM'} at 127.2.35.129:59145; Asking this server to re-register.
08:04:40.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.458796 2682 heartbeater.cc:461] Registering TS with master...
08:04:40.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.462641 3713 leader_election.cc:304] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 91200c15660b4e97b54ebd3774fbcc1c, 9f1831dbc910444c90906336b5d5e1ef; no voters: 8fc933c428a74e04993ae685149e88d5
08:04:40.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.463436 3842 ts_manager.cc:194] Registered new tserver with Master: 847f21d014eb45918a004c118c3fc21d (127.2.35.129:40017)
08:04:40.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.464001 3910 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 4 FOLLOWER]: Leader election won for term 4
08:04:40.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.483165 3910 raft_consensus.cc:697] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [term 4 LEADER]: Becoming Leader. State: Replica: 9f1831dbc910444c90906336b5d5e1ef, State: Running, Role: LEADER
08:04:40.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.484813 3910 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:40.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.500578 3917 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9f1831dbc910444c90906336b5d5e1ef. Latest consensus state: current_term: 4 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:40.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.506935 3917 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: LEADER
08:04:40.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.513866 3921 catalog_manager.cc:1510] Loading table and tablet metadata into memory...
08:04:40.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.535915 3921 catalog_manager.cc:1519] Initializing Kudu cluster ID...
08:04:40.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.541602 3921 catalog_manager.cc:1294] Loaded cluster ID: bed5708457f44b128d8e8c9370b33201
08:04:40.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.546970 3921 catalog_manager.cc:1530] Initializing Kudu internal certificate authority...
08:04:40.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.552045 3921 catalog_manager.cc:1539] Loading token signing keys...
08:04:40.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.560559 3921 catalog_manager.cc:6080] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef: Loaded TSK: 0
08:04:40.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.565980 3921 catalog_manager.cc:1549] Initializing in-progress tserver states...
08:04:40.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.573679 2829 heartbeater.cc:344] Connected to a master server at 127.2.35.189:40817
08:04:40.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.577422 3842 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "a39b773fc5ee472ca6ae106471a57777" instance_seqno: 1781683452002663) as {username='kudu', principal='kudu/127.2.35.130@KRBTEST.COM'} at 127.2.35.130:40847; Asking this server to re-register.
08:04:40.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.582746 2829 heartbeater.cc:461] Registering TS with master...
08:04:40.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.587309 3842 ts_manager.cc:194] Registered new tserver with Master: a39b773fc5ee472ca6ae106471a57777 (127.2.35.130:34455)
08:04:40.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.885602 3852 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [term 4 FOLLOWER]: Refusing update from remote peer 9f1831dbc910444c90906336b5d5e1ef: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
08:04:40.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.889051 3917 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [LEADER]: Connected to new peer: Peer: permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
08:04:40.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.947639 3922 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9f1831dbc910444c90906336b5d5e1ef. Latest consensus state: current_term: 4 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:40.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.948385 3922 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.976459 3910 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:40.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.977240 3910 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef [sys.catalog]: This master's current role is: LEADER
08:04:40.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.983736 3922 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "9f1831dbc910444c90906336b5d5e1ef" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } } }
08:04:40.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:40.984412 3922 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91200c15660b4e97b54ebd3774fbcc1c [sys.catalog]: This master's current role is: FOLLOWER
08:04:41.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:41.017587 3714 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef -> Peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Couldn't send request to peer 8fc933c428a74e04993ae685149e88d5. Status: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
08:04:41.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:41.548873 3686 master_runner.cc:187] Error getting master registration for 127.2.35.190:32797: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111),
08:04:41.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:41.549465 3686 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
08:04:41.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:41 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683481, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:41.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:41.785094 2712 init.cc:295] Successfully reacquired a new kerberos TGT
08:04:42.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:42.175979 2984 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:42.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:42.440579 2683 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:42.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:42.559650 2830 heartbeater.cc:499] Master 127.2.35.188:38047 was elected leader, sending a full tablet report...
08:04:43.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:41.655215 3903 debug-util.cc:398] Leaking SignalData structure 0x7b080002ef00 after lost signal to thread 3878
08:04:43.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:43.009120 3878 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.754s user 0.000s sys 0.002s
08:04:43.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:43.010957 3878 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.756s user 0.000s sys 0.002s
08:04:43.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:40.254820 3905 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:43.012413 3906 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2755 milliseconds
08:04:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:43.012636 3907 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
08:04:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.012588 3878 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
08:04:43.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.017441 3878 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
08:04:43.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.020043 3878 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
08:04:43.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.021410 3878 hybrid_clock.cc:648] HybridClock initialized: now 1781683483021375 us; error 39 us; skew 500 ppm
08:04:43.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.022298 3878 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
08:04:43.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:43 dist-test-slave-rgbw krb5kdc[2202](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683483, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
08:04:43.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.028857 3878 init.cc:377] Logged in from keytab as kudu/127.2.35.190@KRBTEST.COM (short username kudu)
08:04:43.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.034463 3878 webserver.cc:533] Webserver started at http://127.2.35.190:35165/ using document root <none> and password file <none>
08:04:43.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.035560 3878 fs_manager.cc:362] Metadata directory not provided
08:04:43.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.035804 3878 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
08:04:43.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.044898 3878 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
08:04:43.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.050231 3946 log_block_manager.cc:4031] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
08:04:43.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.051592 3878 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.051991 3878 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data,/tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8fc933c428a74e04993ae685149e88d5"
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-06-17 08:03:57 on dist-test-slave-rgbw"
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.054009 3878 fs_report.cc:389] FS layout report
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/wal
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/data
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
08:04:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
08:04:43.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
08:04:43.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
08:04:43.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
08:04:43.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
08:04:43.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.117756 3878 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
08:04:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.123833 3878 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
08:04:43.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.124353 3878 kserver.cc:163] Server-wide thread pool size limit: 3276
08:04:43.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.216625 3878 rpc_server.cc:307] RPC server started. Bound to: 127.2.35.190:32797
08:04:43.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.216794 3997 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.2.35.190:32797 every 8 connection(s)
08:04:43.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.222563 3878 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskexLxKm/test-tmp/mini-kudu-cluster7288218219669161229/master-0/data/info.pb
08:04:43.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.231855 2190 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu as pid 3878
08:04:43.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
08:04:43.233 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.2.35.188:38047(127.2.35.188:38047)] connecting to peer
08:04:43.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.236264 3998 sys_catalog.cc:263] Verifying existing consensus state
08:04:43.238 [DEBUG - kudu-nio-4] (Connection.java:757) [peer master-127.2.35.188:38047(127.2.35.188:38047)] Successfully connected to peer
08:04:43.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.247200 3998 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Bootstrap starting.
08:04:43.252 [DEBUG - kudu-nio-4] (Connection.java:709) [peer master-127.2.35.188:38047(127.2.35.188:38047)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
08:04:43.253 [DEBUG - kudu-nio-4] (Connection.java:239) [peer master-127.2.35.188:38047(127.2.35.188:38047)] handling channelInactive
08:04:43.256 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.35.190:32797(127.2.35.190:32797)] disconnecting while in state TERMINATED
08:04:43.258 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.35.188:38047(127.2.35.188:38047)] disconnecting while in state TERMINATED
08:04:43.259 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.2.35.189:40817(127.2.35.189:40817)] disconnecting while in state TERMINATED
08:04:43.261 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:04:43.273 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:04:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:43.273715 4001 negotiation.cc:336] Failed RPC negotiation. Trace:
08:04:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:43.238441 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:59398 (local address 127.2.35.188:38047)
08:04:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:43.249808 (+ 11367us) server_negotiation.cc:207] Beginning negotiation
08:04:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:43.249846 (+ 38us) server_negotiation.cc:400] Waiting for connection header
08:04:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:43.249886 (+ 40us) server_negotiation.cc:408] Connection header received
08:04:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:43.250155 (+ 269us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
08:04:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:43.250178 (+ 23us) server_negotiation.cc:462] Received NEGOTIATE request from client
08:04:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:43.250379 (+ 201us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
08:04:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:43.250595 (+ 216us) server_negotiation.cc:227] Negotiated authn=SASL
08:04:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0617 08:04:43.272193 (+ 21598us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:59398: BlockingRecv error: recv got EOF from 127.0.0.1:59398 (error 108)
08:04:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":10480,"thread_start_us":10208,"threads_started":1}
08:04:43.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.285457 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 2517
08:04:43.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.313866 3998 log.cc:826] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Log is configured to *not* fsync() on all Append() calls
08:04:43.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:43 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683474, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.131@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:43.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.325601 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 2696
08:04:43.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.361788 2985 heartbeater.cc:344] Connected to a master server at 127.2.35.190:32797
08:04:43.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Jun 17 08:04:43 dist-test-slave-rgbw krb5kdc[2202](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1781683476, etypes {rep=17 tkt=17 ses=17}, kudu/127.2.35.188@KRBTEST.COM for kudu/127.2.35.190@KRBTEST.COM
08:04:43.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.374538 3998 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
08:04:43.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.375547 3998 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5: Bootstrap complete.
08:04:43.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.381938 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 2844
08:04:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.414484 3998 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8fc933c428a74e04993ae685149e88d5" member_type: VOTER last_known_addr { host: "127.2.35.190" port: 32797 } } peers { permanent_uuid: "91200c15660b4e97b54ebd3774fbcc1c" member_type: VOTER last_known_addr { host: "127.2.35.189" port: 40817 } } peers { permanent_uuid: "9f1831dbc910444c90906336b5d5e1ef" member_type: VOTER last_known_addr { host: "127.2.35.188" port: 38047 } }
08:04:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.418480 3998 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8fc933c428a74e04993ae685149e88d5 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8fc933c428a74e04993ae685149e88d5, State: Initialized, Role: FOLLOWER
08:04:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.418612 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 3878
08:04:43.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:43.438899 3714 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.2.35.190:32797 (error 108) [suppressed 8 similar messages]
08:04:43.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.439507 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 3766
08:04:43.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:43.443270 3714 consensus_peers.cc:597] T 00000000000000000000000000000000 P 9f1831dbc910444c90906336b5d5e1ef -> Peer 8fc933c428a74e04993ae685149e88d5 (127.2.35.190:32797): Couldn't send request to peer 8fc933c428a74e04993ae685149e88d5. Status: Network error: Client connection negotiation failed: client connection to 127.2.35.190:32797: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
08:04:43.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:43.469183 2190 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 3686
08:04:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:43.968633 1692 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2380 after lost signal to thread 1623
08:04:43.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260617 08:04:43.969336 1692 debug-util.cc:398] Leaking SignalData structure 0x7b08000adf00 after lost signal to thread 1695
08:04:44.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T08:04:44Z chronyd exiting
08:04:45.719 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
08:04:45.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:45.722651 1461 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 1743
08:04:45.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:45.751178 1461 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 1897
08:04:45.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:45.789131 1461 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 2046
08:04:45.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:45.817649 1461 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 1473
08:04:45.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:45.849648 1461 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 1543
08:04:45.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260617 08:04:45.880573 1461 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskexLxKm/build/tsan/bin/kudu with pid 1618
08:04:47.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-06-17T08:04:47Z chronyd exiting
08:04:48.115 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
08:04:49.501 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
08:04:49.501 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more