15:18:42.558 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:18:42.571 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLsXfcC/test-tmp
15:18:42.574 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:18:42.578 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLsXfcC/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:18:42.902 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:18:43.002 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:18:43.003 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:18:43.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:43.006842 28009 mini_oidc.cc:150] Starting JWKS server
15:18:43.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:43.010283 28009 webserver.cc:492] Webserver started at http://127.0.0.1:36169/ using document root <none> and password file <none>
15:18:43.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:43.010720 28009 mini_oidc.cc:186] Starting OIDC Discovery server
15:18:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:43.012952 28009 webserver.cc:492] Webserver started at http://127.0.0.1:36535/ using document root <none> and password file <none>
15:18:43.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T15:18:43Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:18:43.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T15:18:43Z Disabled control of system clock
15:18:43.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:43.076351 28009 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/wal
15:18:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/data
15:18:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/logs
15:18:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/data/info.pb
15:18:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:18:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.126:34419
15:18:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.126
15:18:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:43.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36169/jwks.json with env {}
15:18:43.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:43.393128 28021 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:43.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:43.393748 28021 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:43.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:43.394310 28021 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:43.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:43.425742 28021 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:18:43.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:43.426041 28021 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:43.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:43.426354 28021 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:18:43.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:43.426699 28021 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:18:43.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:43.426913 28021 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:18:43.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:43.427076 28021 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36169/jwks.json
15:18:43.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:43.462982 28021 master_runner.cc:386] Master server non-default flags:
15:18:43.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:43.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:43.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:43.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:43.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/data
15:18:43.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/wal
15:18:43.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:43.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:43.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:43.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.126:34419
15:18:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:43.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36169/jwks.json
15:18:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/data/info.pb
15:18:43.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.126
15:18:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/logs
15:18:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:43.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:18:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:18:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:18:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:18:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:18:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:43.464550 28021 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:43.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:43.466259 28021 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:18:43.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:43.478466 28027 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:45.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:44.882162 28026 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec80 after lost signal to thread 28021
15:18:45.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.316531 28026 kernel_stack_watchdog.cc:198] Thread 28021 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
15:18:45.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:18:45.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:18:45.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:18:45.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:18:45.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:43.479466 28028 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:45.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.318238 28021 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.839s user 0.739s sys 1.100s
15:18:45.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.319646 28029 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1839 milliseconds
15:18:45.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.320380 28021 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.842s user 0.739s sys 1.100s
15:18:45.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.320510 28030 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:45.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.320724 28021 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:18:45.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.324472 28021 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:45.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.327037 28021 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:45.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.328451 28021 hybrid_clock.cc:648] HybridClock initialized: now 1770045525328404 us; error 52 us; skew 500 ppm
15:18:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.329387 28021 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:45.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.336505 28021 webserver.cc:492] Webserver started at http://127.27.90.126:35439/ using document root <none> and password file <none>
15:18:45.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.337700 28021 fs_manager.cc:362] Metadata directory not provided
15:18:45.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.337983 28021 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:45.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.338574 28021 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:18:45.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.343202 28021 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/data/instance:
15:18:45.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8588480960844c7a9653e04b6ad39270"
15:18:45.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:45 on dist-test-slave-q1dj"
15:18:45.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.344738 28021 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/wal/instance:
15:18:45.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8588480960844c7a9653e04b6ad39270"
15:18:45.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:45 on dist-test-slave-q1dj"
15:18:45.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.353410 28021 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.001s sys 0.007s
15:18:45.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.359716 28037 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:45.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.361574 28021 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
15:18:45.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.362020 28021 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/wal
15:18:45.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8588480960844c7a9653e04b6ad39270"
15:18:45.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:45 on dist-test-slave-q1dj"
15:18:45.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.362507 28021 fs_report.cc:389] FS layout report
15:18:45.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:45.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/wal
15:18:45.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/wal
15:18:45.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/data/data
15:18:45.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:45.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:45.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.414736 28021 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:18:45.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.416427 28021 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:45.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.416978 28021 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:45.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.500428 28021 rpc_server.cc:307] RPC server started. Bound to: 127.27.90.126:34419
15:18:45.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.500495 28088 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.90.126:34419 every 8 connection(s)
15:18:45.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.503438 28021 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/data/info.pb
15:18:45.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.510426 28009 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 28021
15:18:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.510504 28089 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:18:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.510931 28009 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-0/wal/instance
15:18:45.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.514477 28009 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:45.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:45.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/wal
15:18:45.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/data
15:18:45.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:45.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:45.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:45.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/logs
15:18:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/data/info.pb
15:18:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:45.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:45.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:45.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:45.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.125:44071
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.125
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36169/jwks.json with env {}
15:18:45.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.525851 28089 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } has no permanent_uuid. Determining permanent_uuid...
15:18:45.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.556425 28089 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } has no permanent_uuid. Determining permanent_uuid...
15:18:45.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.558964 28039 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.90.125:44071: connect: Connection refused (error 111)
15:18:45.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.563690 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.125:44071: Network error: Client connection negotiation failed: client connection to 127.27.90.125:44071: connect: Connection refused (error 111)
15:18:45.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.613365 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } attempt: 1
15:18:45.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.618295 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.125:44071: Network error: Client connection negotiation failed: client connection to 127.27.90.125:44071: connect: Connection refused (error 111)
15:18:45.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.686717 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } attempt: 2
15:18:45.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.691409 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.125:44071: Network error: Client connection negotiation failed: client connection to 127.27.90.125:44071: connect: Connection refused (error 111)
15:18:45.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.782924 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } attempt: 3
15:18:45.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.787307 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.125:44071: Network error: Client connection negotiation failed: client connection to 127.27.90.125:44071: connect: Connection refused (error 111)
15:18:45.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.867483 28091 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:45.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.868129 28091 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:45.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.868621 28091 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:45.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.901165 28091 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:18:45.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.901480 28091 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:45.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.901672 28091 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:18:45.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.901855 28091 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:18:45.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.902110 28091 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:18:45.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.902364 28091 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36169/jwks.json
15:18:45.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.930771 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } attempt: 4
15:18:45.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.934861 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.125:44071: Network error: Client connection negotiation failed: client connection to 127.27.90.125:44071: connect: Connection refused (error 111)
15:18:45.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.940517 28091 master_runner.cc:386] Master server non-default flags:
15:18:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/data
15:18:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/wal
15:18:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:45.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:45.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:45.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.125:44071
15:18:45.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:45.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:45.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36169/jwks.json
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/data/info.pb
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.125
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/logs
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:18:45.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:18:45.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:18:45.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:45.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:18:45.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:18:45.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:45.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.942103 28091 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:45.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:45.944528 28091 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:18:45.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.958137 28100 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:46.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:46.234306 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } attempt: 5
15:18:46.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:46.249063 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.125:44071: Network error: Client connection negotiation failed: client connection to 127.27.90.125:44071: connect: Connection refused (error 111)
15:18:46.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:46.796622 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } attempt: 6
15:18:46.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:46.825688 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.125:44071: Network error: Client connection negotiation failed: client connection to 127.27.90.125:44071: connect: Connection refused (error 111)
15:18:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:47.361794 28099 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec80 after lost signal to thread 28091
15:18:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:47.763610 28099 kernel_stack_watchdog.cc:198] Thread 28091 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
15:18:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:18:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:18:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:18:47.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:18:47.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:45.958732 28101 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:47.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:47.765367 28091 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.808s user 0.654s sys 1.150s
15:18:47.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:47.765802 28091 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.808s user 0.654s sys 1.150s
15:18:47.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:47.767536 28103 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:47.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:47.770054 28102 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1806 milliseconds
15:18:47.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.770150 28091 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:18:47.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.771351 28091 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:47.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.773862 28091 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:47.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.775281 28091 hybrid_clock.cc:648] HybridClock initialized: now 1770045527775248 us; error 42 us; skew 500 ppm
15:18:47.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.776296 28091 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:47.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.782729 28091 webserver.cc:492] Webserver started at http://127.27.90.125:37377/ using document root <none> and password file <none>
15:18:47.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.783885 28091 fs_manager.cc:362] Metadata directory not provided
15:18:47.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.784148 28091 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:47.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.784731 28091 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:18:47.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.790108 28091 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/data/instance:
15:18:47.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db8a6ccacdc441fa8a953b4394f90b59"
15:18:47.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:47 on dist-test-slave-q1dj"
15:18:47.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.791489 28091 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/wal/instance:
15:18:47.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db8a6ccacdc441fa8a953b4394f90b59"
15:18:47.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:47 on dist-test-slave-q1dj"
15:18:47.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.800606 28091 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.002s
15:18:47.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.807091 28112 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:47.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.808323 28091 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
15:18:47.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.808737 28091 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/wal
15:18:47.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db8a6ccacdc441fa8a953b4394f90b59"
15:18:47.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:47 on dist-test-slave-q1dj"
15:18:47.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.809160 28091 fs_report.cc:389] FS layout report
15:18:47.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:47.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/wal
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/wal
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/data/data
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:47.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:47.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.859139 28091 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:18:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.860926 28091 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:47.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.861486 28091 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:47.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.886317 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } attempt: 7
15:18:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:47.895771 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.125:44071: Network error: Client connection negotiation failed: client connection to 127.27.90.125:44071: connect: Connection refused (error 111)
15:18:47.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.950380 28091 rpc_server.cc:307] RPC server started. Bound to: 127.27.90.125:44071
15:18:47.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.950443 28165 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.90.125:44071 every 8 connection(s)
15:18:47.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.953315 28091 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/data/info.pb
15:18:47.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.956215 28009 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 28091
15:18:47.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.956570 28009 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-1/wal/instance
15:18:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.959602 28009 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/wal
15:18:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/data
15:18:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:47.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/logs
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/data/info.pb
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:18:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.124:38615
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.124
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36169/jwks.json with env {}
15:18:47.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.964780 28166 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:18:47.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:47.979472 28166 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } has no permanent_uuid. Determining permanent_uuid...
15:18:48.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:48.004377 28166 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } has no permanent_uuid. Determining permanent_uuid...
15:18:48.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:48.020519 28166 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } has no permanent_uuid. Determining permanent_uuid...
15:18:48.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.023896 28115 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.028505 28166 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:48.078078 28166 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 1
15:18:48.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.082445 28166 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:48.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:48.150879 28166 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 2
15:18:48.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.155381 28166 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:48.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:48.246861 28166 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 3
15:18:48.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.251418 28166 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:48.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.292550 28168 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:48.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.293110 28168 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:48.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.293586 28168 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.326956 28168 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:18:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.327261 28168 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.327478 28168 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:18:48.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.327668 28168 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:18:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.327965 28168 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:18:48.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.328137 28168 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36169/jwks.json
15:18:48.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:48.365720 28168 master_runner.cc:386] Master server non-default flags:
15:18:48.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:48.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:48.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:48.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:48.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/data
15:18:48.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/wal
15:18:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:18:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:18:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.124:38615
15:18:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:18:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:18:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36169/jwks.json
15:18:48.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/data/info.pb
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.124
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/logs
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:18:48.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:18:48.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:18:48.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:48.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:18:48.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:18:48.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:48.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:48.367106 28168 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:48.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:48.369449 28168 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:18:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.382181 28178 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:48.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:48.394876 28166 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 4
15:18:48.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.404690 28166 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:48.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:48.704252 28166 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 5
15:18:48.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.714459 28166 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:49.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:49.262050 28166 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 6
15:18:49.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:49.292042 28166 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:49.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:49.986392 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } attempt: 8
15:18:50.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.026427 28089 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } has no permanent_uuid. Determining permanent_uuid...
15:18:50.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.048151 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:50.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.113734 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 1
15:18:50.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.122107 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:50.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.175691 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 2
15:18:50.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.185461 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:50.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:49.786897 28177 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec80 after lost signal to thread 28168
15:18:50.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:48.384358 28179 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:50.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.232827 28168 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.849s user 0.660s sys 1.181s
15:18:50.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.235019 28168 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.852s user 0.660s sys 1.181s
15:18:50.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.236498 28181 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:50.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.238759 28180 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1851 milliseconds
15:18:50.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.238806 28168 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:18:50.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.240540 28168 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.243323 28168 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:50.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.244752 28168 hybrid_clock.cc:648] HybridClock initialized: now 1770045530244685 us; error 81 us; skew 500 ppm
15:18:50.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.246317 28168 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:50.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.253700 28168 webserver.cc:492] Webserver started at http://127.27.90.124:43093/ using document root <none> and password file <none>
15:18:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.254886 28168 fs_manager.cc:362] Metadata directory not provided
15:18:50.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.255169 28168 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:50.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.255733 28168 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:18:50.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.260715 28168 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/data/instance:
15:18:50.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba523c76595b41198bb1d149599d5536"
15:18:50.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:50 on dist-test-slave-q1dj"
15:18:50.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.262032 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 3
15:18:50.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.262331 28168 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/wal/instance:
15:18:50.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba523c76595b41198bb1d149599d5536"
15:18:50.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:50 on dist-test-slave-q1dj"
15:18:50.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.266759 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:50.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.272141 28168 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.006s
15:18:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.278224 28193 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.279498 28168 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.005s
15:18:50.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.279925 28168 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/wal
15:18:50.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ba523c76595b41198bb1d149599d5536"
15:18:50.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:50 on dist-test-slave-q1dj"
15:18:50.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.280354 28168 fs_report.cc:389] FS layout report
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/wal
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/wal
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/data/data
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:50.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:50.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.337466 28168 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:18:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.339115 28168 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:50.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.339661 28168 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:50.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.352675 28166 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 7
15:18:50.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.361021 28166 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:50.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.422268 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 4
15:18:50.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.427641 28089 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.90.124:38615: Network error: Client connection negotiation failed: client connection to 127.27.90.124:38615: connect: Connection refused (error 111)
15:18:50.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.428993 28168 rpc_server.cc:307] RPC server started. Bound to: 127.27.90.124:38615
15:18:50.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.429054 28246 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.90.124:38615 every 8 connection(s)
15:18:50.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.431862 28168 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/data/info.pb
15:18:50.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.432931 28009 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 28168
15:18:50.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.433398 28009 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/master-2/wal/instance
15:18:50.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.441833 28247 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:18:50.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.456274 28247 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } has no permanent_uuid. Determining permanent_uuid...
15:18:50.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.485845 28247 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } has no permanent_uuid. Determining permanent_uuid...
15:18:50.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.496611 28247 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } has no permanent_uuid. Determining permanent_uuid...
15:18:50.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.517316 28247 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536: Bootstrap starting.
15:18:50.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.525205 28247 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536: Neither blocks nor log segments found. Creating new log.
15:18:50.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.526994 28247 log.cc:826] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536: Log is configured to *not* fsync() on all Append() calls
15:18:50.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.532847 28247 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536: No bootstrap required, opened a new log
15:18:50.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.553081 28247 raft_consensus.cc:359] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } }
15:18:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.553824 28247 raft_consensus.cc:385] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:18:50.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.554101 28247 raft_consensus.cc:740] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba523c76595b41198bb1d149599d5536, State: Initialized, Role: FOLLOWER
15:18:50.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.554975 28247 consensus_queue.cc:260] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } }
15:18:50.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.557806 28255 sys_catalog.cc:455] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } } }
15:18:50.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.558367 28255 sys_catalog.cc:458] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [sys.catalog]: This master's current role is: FOLLOWER
15:18:50.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.559351 28247 sys_catalog.cc:565] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [sys.catalog]: configured and running, proceeding with master startup.
15:18:50.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.590184 28266 catalog_manager.cc:1568] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:18:50.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.590620 28266 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:18:50.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.595640 28199 tablet.cc:2378] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:18:50.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.585393 28168 master_runner.cc:186] Error getting master registration for 127.27.90.126:34419: OK, instance_id { permanent_uuid: "8588480960844c7a9653e04b6ad39270" instance_seqno: 1770045525477879 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:18:50.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.610170 28168 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:18:50.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.724084 28089 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 5
15:18:50.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.748193 28089 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270: Bootstrap starting.
15:18:50.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.753949 28089 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270: Neither blocks nor log segments found. Creating new log.
15:18:50.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.756300 28089 log.cc:826] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270: Log is configured to *not* fsync() on all Append() calls
15:18:50.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.762789 28089 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270: No bootstrap required, opened a new log
15:18:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.792289 28089 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } }
15:18:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.793668 28089 raft_consensus.cc:385] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:18:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.794333 28089 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8588480960844c7a9653e04b6ad39270, State: Initialized, Role: FOLLOWER
15:18:50.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.796100 28089 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } }
15:18:50.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.800535 28269 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } } }
15:18:50.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.801833 28269 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [sys.catalog]: This master's current role is: FOLLOWER
15:18:50.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.802578 28089 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [sys.catalog]: configured and running, proceeding with master startup.
15:18:50.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.842011 28280 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:18:50.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.842815 28280 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:18:50.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.839195 28021 master_runner.cc:186] Error getting master registration for 127.27.90.125:44071: OK, instance_id { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" instance_seqno: 1770045527926545 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:18:50.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.866444 28021 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:18:50.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.915921 28255 raft_consensus.cc:493] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:18:50.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.916512 28255 raft_consensus.cc:515] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } }
15:18:50.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.920537 28064 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ba523c76595b41198bb1d149599d5536" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8588480960844c7a9653e04b6ad39270" is_pre_election: true
15:18:50.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.922511 28064 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba523c76595b41198bb1d149599d5536 in term 0.
15:18:50.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.925792 28255 leader_election.cc:290] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8588480960844c7a9653e04b6ad39270 (127.27.90.126:34419), db8a6ccacdc441fa8a953b4394f90b59 (127.27.90.125:44071)
15:18:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.927182 28194 leader_election.cc:304] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8588480960844c7a9653e04b6ad39270, ba523c76595b41198bb1d149599d5536; no voters:
15:18:50.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:50.928813 28043 tablet.cc:2378] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:18:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.928434 28255 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:18:50.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.930347 28255 raft_consensus.cc:493] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:18:50.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.930722 28255 raft_consensus.cc:3060] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [term 0 FOLLOWER]: Advancing to term 1
15:18:50.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.938230 28255 raft_consensus.cc:515] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } }
15:18:50.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.940354 28255 leader_election.cc:290] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [CANDIDATE]: Term 1 election: Requested vote from peers 8588480960844c7a9653e04b6ad39270 (127.27.90.126:34419), db8a6ccacdc441fa8a953b4394f90b59 (127.27.90.125:44071)
15:18:50.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.941572 28064 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ba523c76595b41198bb1d149599d5536" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8588480960844c7a9653e04b6ad39270"
15:18:50.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.942183 28064 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [term 0 FOLLOWER]: Advancing to term 1
15:18:50.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.950279 28064 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba523c76595b41198bb1d149599d5536 in term 1.
15:18:50.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.952394 28194 leader_election.cc:304] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8588480960844c7a9653e04b6ad39270, ba523c76595b41198bb1d149599d5536; no voters:
15:18:50.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.953966 28255 raft_consensus.cc:2804] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [term 1 FOLLOWER]: Leader election won for term 1
15:18:50.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.984664 28255 raft_consensus.cc:697] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [term 1 LEADER]: Becoming Leader. State: Replica: ba523c76595b41198bb1d149599d5536, State: Running, Role: LEADER
15:18:50.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:50.986284 28255 consensus_queue.cc:237] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } }
15:18:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.008698 28282 sys_catalog.cc:455] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ba523c76595b41198bb1d149599d5536. Latest consensus state: current_term: 1 leader_uuid: "ba523c76595b41198bb1d149599d5536" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } } }
15:18:51.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.013056 28282 sys_catalog.cc:458] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [sys.catalog]: This master's current role is: LEADER
15:18:51.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.018397 28284 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:18:51.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.035650 28284 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:18:51.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.065146 28064 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [term 1 FOLLOWER]: Refusing update from remote peer ba523c76595b41198bb1d149599d5536: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:18:51.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.067274 28282 consensus_queue.cc:1048] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
15:18:51.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.104163 28285 mvcc.cc:204] Tried to move back new op lower bound from 7250106495222116352 to 7250106494932975616. Current Snapshot: MvccSnapshot[applied={T|T < 7250106495222116352}]
15:18:51.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.110031 28269 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ba523c76595b41198bb1d149599d5536. Latest consensus state: current_term: 1 leader_uuid: "ba523c76595b41198bb1d149599d5536" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } } }
15:18:51.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.111214 28269 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [sys.catalog]: This master's current role is: FOLLOWER
15:18:51.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.113438 28255 sys_catalog.cc:455] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ba523c76595b41198bb1d149599d5536" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } } }
15:18:51.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.114200 28255 sys_catalog.cc:458] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [sys.catalog]: This master's current role is: LEADER
15:18:51.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.121862 28284 catalog_manager.cc:1357] Generated new cluster ID: 25dd53f3d7a44abd98fc6b94f3442590
15:18:51.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.122193 28284 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:18:51.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.129057 28269 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ba523c76595b41198bb1d149599d5536" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } } }
15:18:51.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.129799 28269 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270 [sys.catalog]: This master's current role is: FOLLOWER
15:18:51.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.175450 28284 catalog_manager.cc:1380] Generated new certificate authority record
15:18:51.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.179409 28284 catalog_manager.cc:1514] Loading token signing keys...
15:18:51.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.213420 28284 catalog_manager.cc:6027] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536: Generated new TSK 0
15:18:51.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.216406 28284 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.250593 28009 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/wal
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/data
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/logs
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/data/info.pb
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:51.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.65:0
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.90.65
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.65
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:51.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:51.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:18:51.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.850097 28280 catalog_manager.cc:1269] Loaded cluster ID: 25dd53f3d7a44abd98fc6b94f3442590
15:18:51.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.850922 28280 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270: loading cluster ID for follower catalog manager: success
15:18:51.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.869237 28280 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270: acquiring CA information for follower catalog manager: success
15:18:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:51.894649 28280 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 8588480960844c7a9653e04b6ad39270: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:18:51.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:51.947589 28291 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:51.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:51.948261 28291 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:51.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:51.949018 28291 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:52.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:52.011796 28291 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:52.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:52.013332 28291 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.90.65
15:18:52.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:52.086067 28291 tablet_server_runner.cc:78] Tablet server non-default flags:
15:18:52.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:52.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:52.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/data
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/wal
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.65:0
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/data/info.pb
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.65
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.90.65
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/logs
15:18:52.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:18:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:18:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:18:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:18:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:18:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:52.088049 28291 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:52.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:52.090451 28291 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:18:52.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:52.117703 28299 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:52.451443 28166 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } attempt: 8
15:18:52.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:52.695381 28166 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59: Bootstrap starting.
15:18:52.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:52.871337 28166 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59: Neither blocks nor log segments found. Creating new log.
15:18:52.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:52.906976 28166 log.cc:826] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59: Log is configured to *not* fsync() on all Append() calls
15:18:53.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.021878 28166 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59: No bootstrap required, opened a new log
15:18:53.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:53.128885 28195 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.27.90.125:44071 timed out after 2.209s (SENT)
15:18:53.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.522521 28166 raft_consensus.cc:359] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } }
15:18:53.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.523902 28166 raft_consensus.cc:385] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:18:53.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.524397 28166 raft_consensus.cc:740] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: db8a6ccacdc441fa8a953b4394f90b59, State: Initialized, Role: FOLLOWER
15:18:53.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.525842 28166 consensus_queue.cc:260] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } }
15:18:53.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.559242 28313 sys_catalog.cc:455] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } } }
15:18:53.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.560572 28313 sys_catalog.cc:458] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [sys.catalog]: This master's current role is: FOLLOWER
15:18:53.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.565397 28166 sys_catalog.cc:565] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [sys.catalog]: configured and running, proceeding with master startup.
15:18:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.565605 28137 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ba523c76595b41198bb1d149599d5536" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db8a6ccacdc441fa8a953b4394f90b59"
15:18:53.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.566543 28137 raft_consensus.cc:3060] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [term 0 FOLLOWER]: Advancing to term 1
15:18:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.568106 28140 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ba523c76595b41198bb1d149599d5536" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db8a6ccacdc441fa8a953b4394f90b59"
15:18:53.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:53.569355 28140 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:42102 (request call id 2) took 2627 ms (client timeout 2240 ms). Trace:
15:18:53.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:53.569841 28140 rpcz_store.cc:269] 0202 15:18:50.941583 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:18:53.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:18:50.941791 (+ 208us) service_pool.cc:225] Handling call
15:18:53.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:18:53.569253 (+2627462us) inbound_call.cc:173] Queueing success response
15:18:53.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:18:53.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.571180 28141 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ba523c76595b41198bb1d149599d5536" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db8a6ccacdc441fa8a953b4394f90b59" is_pre_election: true
15:18:53.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.572147 28138 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ba523c76595b41198bb1d149599d5536" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "db8a6ccacdc441fa8a953b4394f90b59" is_pre_election: true
15:18:53.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:53.573701 28141 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:42102 (request call id 1) took 2652 ms (client timeout 2209 ms). Trace:
15:18:53.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:53.574190 28141 rpcz_store.cc:269] 0202 15:18:50.921235 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:18:53.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:18:50.921381 (+ 146us) service_pool.cc:225] Handling call
15:18:53.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:18:53.573614 (+2652233us) inbound_call.cc:173] Queueing success response
15:18:53.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:18:53.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.598054 28137 raft_consensus.cc:2468] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba523c76595b41198bb1d149599d5536 in term 1.
15:18:53.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.599543 28139 raft_consensus.cc:1275] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [term 1 FOLLOWER]: Refusing update from remote peer ba523c76595b41198bb1d149599d5536: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:18:53.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.601869 28139 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:42102 (request call id 3) took 2538 ms. Trace:
15:18:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.602325 28139 rpcz_store.cc:276] 0202 15:18:51.063201 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:18:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:18:51.063390 (+ 189us) service_pool.cc:225] Handling call
15:18:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:18:53.591054 (+2527664us) spinlock_profiling.cc:243] Waited 6.85 ms on lock 0x7b54000303a8. stack: 00007f26b20d7373 00007f26b20d7270 00007f26b1c4c574 0000000000508744 00000000005086d9 00007f26ba153e21 00007f26ba1ceeec 00007f26ba203bbc 00007f26ba1ee46b 00007f26ba1edf02 00007f26b88a2d34 00007f26b88a2cc2 00007f26b88a2bf1 00007f26b88a2b6c 00007f26b88a1e22 00007f26b6ebd34c
15:18:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:18:53.598804 (+ 7750us) raft_consensus.cc:1471] Updating replica for 0 ops
15:18:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:18:53.600027 (+ 1223us) raft_consensus.cc:1697] Filling consensus response to leader.
15:18:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:18:53.601500 (+ 1473us) spinlock_profiling.cc:243] Waited 7.45 ms on lock 0x7b5400030550. stack: 00007f26b20d7373 00007f26b20d7270 00007f26b1c4c574 0000000000508744 00000000005086d9 00007f26b91285a1 00007f26b91be52d 00007f26ba1edf49 00007f26b88a2d34 00007f26b88a2cc2 00007f26b88a2bf1 00007f26b88a2b6c 00007f26b88a1e22 00007f26b6ebd34c 00007f26b6ebc786 00007f26b6ebc12f
15:18:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:18:53.601784 (+ 284us) inbound_call.cc:173] Queueing success response
15:18:53.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":32900736}
15:18:53.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.604779 28305 consensus_queue.cc:1048] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [LEADER]: Connected to new peer: Peer: permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:18:53.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:53.623015 28118 tablet.cc:2378] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:18:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.702224 28313 sys_catalog.cc:455] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ba523c76595b41198bb1d149599d5536. Latest consensus state: current_term: 1 leader_uuid: "ba523c76595b41198bb1d149599d5536" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } } }
15:18:53.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.703532 28313 sys_catalog.cc:458] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [sys.catalog]: This master's current role is: FOLLOWER
15:18:53.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.702332 28320 sys_catalog.cc:455] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "ba523c76595b41198bb1d149599d5536" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } } }
15:18:53.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.709856 28320 sys_catalog.cc:458] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59 [sys.catalog]: This master's current role is: FOLLOWER
15:18:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.767422 28330 catalog_manager.cc:1269] Loaded cluster ID: 25dd53f3d7a44abd98fc6b94f3442590
15:18:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.767920 28330 catalog_manager.cc:1562] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59: loading cluster ID for follower catalog manager: success
15:18:53.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.779649 28330 catalog_manager.cc:1584] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59: acquiring CA information for follower catalog manager: success
15:18:53.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:53.788484 28330 catalog_manager.cc:1612] T 00000000000000000000000000000000 P db8a6ccacdc441fa8a953b4394f90b59: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:18:54.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.217423 28305 sys_catalog.cc:455] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "ba523c76595b41198bb1d149599d5536" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8588480960844c7a9653e04b6ad39270" member_type: VOTER last_known_addr { host: "127.27.90.126" port: 34419 } } peers { permanent_uuid: "db8a6ccacdc441fa8a953b4394f90b59" member_type: VOTER last_known_addr { host: "127.27.90.125" port: 44071 } } peers { permanent_uuid: "ba523c76595b41198bb1d149599d5536" member_type: VOTER last_known_addr { host: "127.27.90.124" port: 38615 } } }
15:18:54.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.218386 28305 sys_catalog.cc:458] T 00000000000000000000000000000000 P ba523c76595b41198bb1d149599d5536 [sys.catalog]: This master's current role is: LEADER
15:18:54.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:53.515177 28298 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 28291
15:18:54.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:54.802488 28298 kernel_stack_watchdog.cc:198] Thread 28291 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 389ms:
15:18:54.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:18:54.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:18:54.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:18:54.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:18:54.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:52.126437 28300 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:54.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:54.803990 28291 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.681s user 0.567s sys 0.891s
15:18:54.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:54.804219 28301 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2680 milliseconds
15:18:54.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:54.805125 28291 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.682s user 0.567s sys 0.892s
15:18:54.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:54.809281 28302 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:54.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.809347 28291 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:18:54.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.810846 28291 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:54.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.813616 28291 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:54.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.815085 28291 hybrid_clock.cc:648] HybridClock initialized: now 1770045534814980 us; error 116 us; skew 500 ppm
15:18:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.816193 28291 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:54.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.824975 28291 webserver.cc:492] Webserver started at http://127.27.90.65:35071/ using document root <none> and password file <none>
15:18:54.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.826265 28291 fs_manager.cc:362] Metadata directory not provided
15:18:54.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.826578 28291 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:54.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.827103 28291 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:18:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.833595 28291 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/data/instance:
15:18:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4e1492e3dd444235b13630d626179d6e"
15:18:54.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:54 on dist-test-slave-q1dj"
15:18:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.835248 28291 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/wal/instance:
15:18:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4e1492e3dd444235b13630d626179d6e"
15:18:54.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:54 on dist-test-slave-q1dj"
15:18:54.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.845433 28291 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.002s sys 0.006s
15:18:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.853830 28338 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.855330 28291 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.004s
15:18:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.855760 28291 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/wal
15:18:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4e1492e3dd444235b13630d626179d6e"
15:18:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:54 on dist-test-slave-q1dj"
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.856256 28291 fs_report.cc:389] FS layout report
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/wal
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/wal
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/data/data
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:54.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.923786 28291 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:18:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.925870 28291 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:54.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.926460 28291 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:54.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.930402 28291 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:18:54.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.935400 28291 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:18:54.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.935631 28291 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:18:54.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.935923 28291 ts_tablet_manager.cc:616] Registered 0 tablets
15:18:54.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:54.936105 28291 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:18:55.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.132797 28291 rpc_server.cc:307] RPC server started. Bound to: 127.27.90.65:41489
15:18:55.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.132858 28450 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.90.65:41489 every 8 connection(s)
15:18:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.136940 28291 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/data/info.pb
15:18:55.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.139266 28009 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 28291
15:18:55.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.142055 28009 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-0/wal/instance
15:18:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.169773 28009 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/wal
15:18:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/data
15:18:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/logs
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/data/info.pb
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.66:0
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.90.66
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.66
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:55.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:55.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:55.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:55.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:18:55.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.202785 28451 heartbeater.cc:344] Connected to a master server at 127.27.90.124:38615
15:18:55.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.203332 28451 heartbeater.cc:461] Registering TS with master...
15:18:55.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.204792 28451 heartbeater.cc:507] Master 127.27.90.124:38615 requested a full tablet report, sending...
15:18:55.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.208768 28212 ts_manager.cc:194] Registered new tserver with Master: 4e1492e3dd444235b13630d626179d6e (127.27.90.65:41489)
15:18:55.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.212085 28212 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.90.65:49779
15:18:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.247200 28456 heartbeater.cc:344] Connected to a master server at 127.27.90.126:34419
15:18:55.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.247627 28456 heartbeater.cc:461] Registering TS with master...
15:18:55.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.248541 28456 heartbeater.cc:507] Master 127.27.90.126:34419 requested a full tablet report, sending...
15:18:55.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.251941 28053 ts_manager.cc:194] Registered new tserver with Master: 4e1492e3dd444235b13630d626179d6e (127.27.90.65:41489)
15:18:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.255903 28452 heartbeater.cc:344] Connected to a master server at 127.27.90.125:44071
15:18:55.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.256317 28452 heartbeater.cc:461] Registering TS with master...
15:18:55.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.257022 28452 heartbeater.cc:507] Master 127.27.90.125:44071 requested a full tablet report, sending...
15:18:55.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.260157 28131 ts_manager.cc:194] Registered new tserver with Master: 4e1492e3dd444235b13630d626179d6e (127.27.90.65:41489)
15:18:55.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:55.582257 28455 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:55.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:55.582752 28455 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:55.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:55.583189 28455 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:55.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:55.620277 28455 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:55.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:55.621770 28455 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.90.66
15:18:55.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.659121 28455 tablet_server_runner.cc:78] Tablet server non-default flags:
15:18:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/data
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/wal
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.66:0
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/data/info.pb
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.66
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.90.66
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/logs
15:18:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:18:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:18:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:18:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:18:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:18:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.660533 28455 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:55.662654 28455 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:18:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:55.678267 28466 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.219012 28451 heartbeater.cc:499] Master 127.27.90.124:38615 was elected leader, sending a full tablet report...
15:18:56.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:55.679999 28467 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:56.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:55.684201 28469 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:56.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:56.924539 28468 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:18:56.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.924665 28455 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:18:56.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.928757 28455 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:56.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.931607 28455 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:56.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.933251 28455 hybrid_clock.cc:648] HybridClock initialized: now 1770045536933214 us; error 79 us; skew 500 ppm
15:18:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.934144 28455 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:56.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.941645 28455 webserver.cc:492] Webserver started at http://127.27.90.66:39717/ using document root <none> and password file <none>
15:18:56.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.942675 28455 fs_manager.cc:362] Metadata directory not provided
15:18:56.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.942901 28455 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:56.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.943324 28455 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:18:56.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.948056 28455 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/data/instance:
15:18:56.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d398c3ee6ec4f84b50cd5bfcae51ff9"
15:18:56.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:56 on dist-test-slave-q1dj"
15:18:56.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.949512 28455 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/wal/instance:
15:18:56.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d398c3ee6ec4f84b50cd5bfcae51ff9"
15:18:56.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:56 on dist-test-slave-q1dj"
15:18:56.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.958500 28455 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
15:18:56.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.966413 28478 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.967898 28455 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.003s
15:18:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.968286 28455 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/wal
15:18:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2d398c3ee6ec4f84b50cd5bfcae51ff9"
15:18:56.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:56 on dist-test-slave-q1dj"
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:56.968608 28455 fs_report.cc:389] FS layout report
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/wal
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/wal
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/data/data
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:56.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:57.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.019577 28455 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:18:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.021703 28455 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:57.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.022166 28455 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:57.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.024986 28455 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:18:57.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.030540 28455 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:18:57.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.030778 28455 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:18:57.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.030980 28455 ts_tablet_manager.cc:616] Registered 0 tablets
15:18:57.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.031113 28455 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:18:57.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.199731 28455 rpc_server.cc:307] RPC server started. Bound to: 127.27.90.66:40943
15:18:57.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.199843 28590 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.90.66:40943 every 8 connection(s)
15:18:57.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.202450 28455 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/data/info.pb
15:18:57.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.206177 28009 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 28455
15:18:57.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.206704 28009 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-1/wal/instance
15:18:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.225101 28009 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:18:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/wal
15:18:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/data
15:18:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:18:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:18:57.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:18:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/logs
15:18:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/data/info.pb
15:18:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:18:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:18:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:18:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:18:57.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:18:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:18:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.67:0
15:18:57.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.90.67
15:18:57.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.67
15:18:57.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:57.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:57.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:57.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:57.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:57.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:18:57.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.273346 28592 heartbeater.cc:344] Connected to a master server at 127.27.90.125:44071
15:18:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.273499 28593 heartbeater.cc:344] Connected to a master server at 127.27.90.126:34419
15:18:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.273933 28592 heartbeater.cc:461] Registering TS with master...
15:18:57.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.274371 28593 heartbeater.cc:461] Registering TS with master...
15:18:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.275411 28592 heartbeater.cc:507] Master 127.27.90.125:44071 requested a full tablet report, sending...
15:18:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.275458 28593 heartbeater.cc:507] Master 127.27.90.126:34419 requested a full tablet report, sending...
15:18:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.276016 28591 heartbeater.cc:344] Connected to a master server at 127.27.90.124:38615
15:18:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.276355 28591 heartbeater.cc:461] Registering TS with master...
15:18:57.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.277850 28591 heartbeater.cc:507] Master 127.27.90.124:38615 requested a full tablet report, sending...
15:18:57.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.279187 28053 ts_manager.cc:194] Registered new tserver with Master: 2d398c3ee6ec4f84b50cd5bfcae51ff9 (127.27.90.66:40943)
15:18:57.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.279821 28131 ts_manager.cc:194] Registered new tserver with Master: 2d398c3ee6ec4f84b50cd5bfcae51ff9 (127.27.90.66:40943)
15:18:57.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.281838 28212 ts_manager.cc:194] Registered new tserver with Master: 2d398c3ee6ec4f84b50cd5bfcae51ff9 (127.27.90.66:40943)
15:18:57.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.283599 28212 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.90.66:41715
15:18:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:57.630100 28601 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:18:57.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:57.630577 28601 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:18:57.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:57.631033 28601 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:18:57.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:57.662995 28601 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:18:57.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:57.663834 28601 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.90.67
15:18:57.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.700201 28601 tablet_server_runner.cc:78] Tablet server non-default flags:
15:18:57.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:18:57.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.90.84:36325
15:18:57.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:18:57.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:18:57.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/data
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/wal
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.90.67:0
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/data/info.pb
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.90.67
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.90.126:34419,127.27.90.125:44071,127.27.90.124:38615
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.90.67
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/logs
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:18:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.701637 28601 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:18:57.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:57.703418 28601 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:18:57.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:57.718021 28607 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:58.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:58.287883 28591 heartbeater.cc:499] Master 127.27.90.124:38615 was elected leader, sending a full tablet report...
15:18:59.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:57.718518 28608 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:59.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:59.121356 28606 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 28601
15:18:59.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:59.200814 28610 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:18:59.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:59.202343 28609 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1481 milliseconds
15:18:59.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:59.198508 28601 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.480s user 0.497s sys 0.930s
15:18:59.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:59.202915 28601 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.485s user 0.497s sys 0.930s
15:18:59.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.203294 28601 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:18:59.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.208007 28601 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:18:59.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.210427 28601 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:18:59.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.212225 28601 hybrid_clock.cc:648] HybridClock initialized: now 1770045539212128 us; error 94 us; skew 500 ppm
15:18:59.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.213184 28601 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:18:59.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.219794 28601 webserver.cc:492] Webserver started at http://127.27.90.67:37823/ using document root <none> and password file <none>
15:18:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.220954 28601 fs_manager.cc:362] Metadata directory not provided
15:18:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.221217 28601 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:18:59.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.221781 28601 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:18:59.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.226322 28601 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/data/instance:
15:18:59.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17ff47229628495f883327a3f08a4ede"
15:18:59.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:59 on dist-test-slave-q1dj"
15:18:59.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.227615 28601 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/wal/instance:
15:18:59.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17ff47229628495f883327a3f08a4ede"
15:18:59.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:59 on dist-test-slave-q1dj"
15:18:59.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.235374 28601 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
15:18:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.241600 28620 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:18:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.242779 28601 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
15:18:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.243180 28601 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/wal
15:18:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17ff47229628495f883327a3f08a4ede"
15:18:59.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:18:59 on dist-test-slave-q1dj"
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.243587 28601 fs_report.cc:389] FS layout report
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/wal
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/wal
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/data/data
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:18:59.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:18:59.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.293314 28601 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:18:59.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.294967 28601 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:18:59.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.295518 28601 kserver.cc:163] Server-wide thread pool size limit: 3276
15:18:59.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.299232 28601 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:18:59.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.304111 28601 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:18:59.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.304382 28601 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:18:59.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.304700 28601 ts_tablet_manager.cc:616] Registered 0 tablets
15:18:59.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.304895 28601 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:18:59.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:59.405488 28447 debug-util.cc:398] Leaking SignalData structure 0x7b080003e100 after lost signal to thread 28294
15:18:59.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:18:59.406495 28447 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9d00 after lost signal to thread 28450
15:18:59.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.493086 28601 rpc_server.cc:307] RPC server started. Bound to: 127.27.90.67:38143
15:18:59.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.493207 28732 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.90.67:38143 every 8 connection(s)
15:18:59.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.495852 28601 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/data/info.pb
15:18:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.506007 28009 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 28601
15:18:59.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.506603 28009 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster8706870210256346476/ts-2/wal/instance
15:18:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.557981 28735 heartbeater.cc:344] Connected to a master server at 127.27.90.126:34419
15:18:59.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.558552 28735 heartbeater.cc:461] Registering TS with master...
15:18:59.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.560060 28735 heartbeater.cc:507] Master 127.27.90.126:34419 requested a full tablet report, sending...
15:18:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.562801 28734 heartbeater.cc:344] Connected to a master server at 127.27.90.125:44071
15:18:59.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.563211 28734 heartbeater.cc:461] Registering TS with master...
15:18:59.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.564204 28734 heartbeater.cc:507] Master 127.27.90.125:44071 requested a full tablet report, sending...
15:18:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.565475 28733 heartbeater.cc:344] Connected to a master server at 127.27.90.124:38615
15:18:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.565858 28733 heartbeater.cc:461] Registering TS with master...
15:18:59.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.566092 28053 ts_manager.cc:194] Registered new tserver with Master: 17ff47229628495f883327a3f08a4ede (127.27.90.67:38143)
15:18:59.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.566781 28733 heartbeater.cc:507] Master 127.27.90.124:38615 requested a full tablet report, sending...
15:18:59.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.570024 28212 ts_manager.cc:194] Registered new tserver with Master: 17ff47229628495f883327a3f08a4ede (127.27.90.67:38143)
15:18:59.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.572111 28212 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.27.90.67:50907
15:18:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.576421 28130 ts_manager.cc:194] Registered new tserver with Master: 17ff47229628495f883327a3f08a4ede (127.27.90.67:38143)
15:18:59.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:18:59.589089 28009 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
15:18:59.601 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:18:59.603 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:18:59.621 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.90.126"
port: 34419
}
bound_http_address {
host: "127.27.90.126"
port: 35439
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.90.125"
port: 44071
}
bound_http_address {
host: "127.27.90.125"
port: 37377
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.90.124"
port: 38615
}
bound_http_address {
host: "127.27.90.124"
port: 43093
}
}
}
15:18:59.627 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:18:59.631 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.90.65"
port: 41489
}
bound_http_address {
host: "127.27.90.65"
port: 35071
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.90.66"
port: 40943
}
bound_http_address {
host: "127.27.90.66"
port: 39717
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.90.67"
port: 38143
}
bound_http_address {
host: "127.27.90.67"
port: 37823
}
}
}
15:18:59.631 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:18:59.884 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
15:18:59.924 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:19:00.076 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskLsXfcC/test-tmp
15:19:00.077 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:19:00.077 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskLsXfcC/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:19:00.080 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:19:00.528 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:00.528 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:19:00.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:00.533455 28744 mini_oidc.cc:150] Starting JWKS server
15:19:00.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:00.540021 28744 webserver.cc:492] Webserver started at http://127.0.0.1:40711/ using document root <none> and password file <none>
15:19:00.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:00.540659 28744 mini_oidc.cc:186] Starting OIDC Discovery server
15:19:00.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:00.543661 28744 webserver.cc:492] Webserver started at http://127.0.0.1:44479/ using document root <none> and password file <none>
15:19:00.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:00.578294 28733 heartbeater.cc:499] Master 127.27.90.124:38615 was elected leader, sending a full tablet report...
15:19:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:19:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/principal' for realm 'KRBTEST.COM',
15:19:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:19:00.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:00 dist-test-slave-q1dj krb5kdc[28756](info): setting up network...
15:19:00.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:19:00.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:00 dist-test-slave-q1dj krb5kdc[28756](info): set up 2 sockets
15:19:00.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:00 dist-test-slave-q1dj krb5kdc[28756](info): commencing operation
15:19:00.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:19:01.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:01.276500 28729 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad060 after lost signal to thread 28602
15:19:01.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:01.278287 28729 debug-util.cc:398] Leaking SignalData structure 0x7b08000adf40 after lost signal to thread 28732
15:19:02.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:02.504572 28085 debug-util.cc:398] Leaking SignalData structure 0x7b080003fe80 after lost signal to thread 28022
15:19:02.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:02.505551 28085 debug-util.cc:398] Leaking SignalData structure 0x7b0800082420 after lost signal to thread 28088
15:19:02.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:02.879487 28744 mini_kdc.cc:121] Time spent starting KDC: real 2.288s user 0.007s sys 0.015s
15:19:02.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:19:02.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:19:02.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:19:02.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:19:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:19:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:19:02.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:19:02.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:19:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:19:03.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:19:03.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/test-user.keytab.
15:19:03.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/test-user.keytab.
15:19:03.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:03 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045543, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:03.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:19:03.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T15:19:03Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:19:03.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T15:19:03Z Disabled control of system clock
15:19:03.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.18.62@KRBTEST.COM; defaulting to no policy
15:19:03.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:03.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.18.62@KRBTEST.COM" created.
15:19:03.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:03.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:03.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:03.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.18.62@KRBTEST.COM; defaulting to no policy
15:19:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:03.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.18.62@KRBTEST.COM" created.
15:19:03.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:03.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:03.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:03.143530 28744 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/logs
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data/info.pb
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.62:43993
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.62
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.62
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:19:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:19:03.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.565950 28774 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:19:03.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.567652 28774 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:19:03.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.568673 28774 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:19:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.607714 28774 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:19:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.608103 28774 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:19:03.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.608502 28774 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:19:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.608861 28774 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:19:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.609186 28774 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:19:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.609508 28774 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:19:03.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.609772 28774 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40711/jwks.json
15:19:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:03.663054 28774 master_runner.cc:386] Master server non-default flags:
15:19:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:03.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data
15:19:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal
15:19:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:19:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.62
15:19:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:03.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.62:43993
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data/info.pb
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.62
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/logs
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:19:03.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:19:03.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:19:03.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:19:03.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:19:03.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:19:03.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:19:03.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:19:03.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:03.665042 28774 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:19:03.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:03.666707 28774 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:19:03.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.681677 28782 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:05.094161 28781 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 28774
15:19:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:05.530292 28781 kernel_stack_watchdog.cc:198] Thread 28774 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 403ms:
15:19:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:19:05.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:19:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:19:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:19:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:03.686339 28783 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:05.531224 28774 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.843s user 0.477s sys 0.903s
15:19:05.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:05.532084 28774 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.844s user 0.478s sys 0.903s
15:19:05.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:05.533444 28786 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.534951 28774 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:19:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:05.535128 28784 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1844 milliseconds
15:19:05.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.539996 28774 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:19:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.543642 28774 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:19:05.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.545516 28774 hybrid_clock.cc:648] HybridClock initialized: now 1770045545545365 us; error 184 us; skew 500 ppm
15:19:05.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.546893 28774 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:19:05.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:05 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045545, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:05.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.554749 28774 init.cc:377] Logged in from keytab as kudu/127.28.18.62@KRBTEST.COM (short username kudu)
15:19:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.578626 28774 webserver.cc:492] Webserver started at http://127.28.18.62:36881/ using document root <none> and password file <none>
15:19:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.580113 28774 fs_manager.cc:362] Metadata directory not provided
15:19:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.580440 28774 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:19:05.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.581036 28774 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:19:05.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.585887 28774 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data/instance:
15:19:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f40696aca8e42b1bd0548ce27fbc279"
15:19:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:05 on dist-test-slave-q1dj"
15:19:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.587455 28774 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal/instance:
15:19:05.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f40696aca8e42b1bd0548ce27fbc279"
15:19:05.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:05 on dist-test-slave-q1dj"
15:19:05.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.596410 28774 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
15:19:05.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.605195 28794 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:19:05.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.606691 28774 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.003s sys 0.001s
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.607147 28774 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f40696aca8e42b1bd0548ce27fbc279"
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:05 on dist-test-slave-q1dj"
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.607985 28774 fs_report.cc:389] FS layout report
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data/data
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:19:05.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:19:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:19:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:19:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:19:05.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:19:05.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.664491 28774 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:19:05.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.671216 28774 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:19:05.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.671732 28774 kserver.cc:163] Server-wide thread pool size limit: 3276
15:19:05.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.761253 28774 rpc_server.cc:307] RPC server started. Bound to: 127.28.18.62:43993
15:19:05.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.761377 28845 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.18.62:43993 every 8 connection(s)
15:19:05.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.764287 28774 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data/info.pb
15:19:05.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.772051 28846 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:19:05.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.774748 28744 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 28774
15:19:05.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.775308 28744 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal/instance
15:19:05.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.788882 28846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } has no permanent_uuid. Determining permanent_uuid...
15:19:05.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.18.61@KRBTEST.COM; defaulting to no policy
15:19:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.18.61@KRBTEST.COM" created.
15:19:05.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:05 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045545, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.62@KRBTEST.COM for kudu/127.28.18.62@KRBTEST.COM
15:19:05.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.917917 28846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } has no permanent_uuid. Determining permanent_uuid...
15:19:05.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:05.920490 28797 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.18.61:46833: connect: Connection refused (error 111)
15:19:05.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:05.924564 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.61:46833: Network error: Client connection negotiation failed: client connection to 127.28.18.61:46833: connect: Connection refused (error 111)
15:19:05.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:05.974116 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } attempt: 1
15:19:05.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:05.979000 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.61:46833: Network error: Client connection negotiation failed: client connection to 127.28.18.61:46833: connect: Connection refused (error 111)
15:19:06.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:06.047401 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } attempt: 2
15:19:06.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:06.051578 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.61:46833: Network error: Client connection negotiation failed: client connection to 127.28.18.61:46833: connect: Connection refused (error 111)
15:19:06.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:06.143112 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } attempt: 3
15:19:06.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:06.147370 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.61:46833: Network error: Client connection negotiation failed: client connection to 127.28.18.61:46833: connect: Connection refused (error 111)
15:19:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:06.290820 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } attempt: 4
15:19:06.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:06.294993 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.61:46833: Network error: Client connection negotiation failed: client connection to 127.28.18.61:46833: connect: Connection refused (error 111)
15:19:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:06.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:06.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:06.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:06.518342 28744 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.28.18.61: real 0.738s user 0.003s sys 0.009s
15:19:06.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.18.61@KRBTEST.COM; defaulting to no policy
15:19:06.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:06.594497 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } attempt: 5
15:19:06.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:06.599107 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.61:46833: Network error: Client connection negotiation failed: client connection to 127.28.18.61:46833: connect: Connection refused (error 111)
15:19:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.18.61@KRBTEST.COM" created.
15:19:07.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:07.146550 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } attempt: 6
15:19:07.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:07.153106 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.61:46833: Network error: Client connection negotiation failed: client connection to 127.28.18.61:46833: connect: Connection refused (error 111)
15:19:07.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:07.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:07.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:07.866240 28744 mini_kdc.cc:268] Time spent creating service keytab for HTTP/127.28.18.61: real 1.347s user 0.001s sys 0.009s
15:19:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:07.868934 28744 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal
15:19:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data
15:19:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:19:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:19:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:19:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/logs
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data/info.pb
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:07.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.61:46833
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.61
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.61
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:07.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:19:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:19:08.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:08.213547 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } attempt: 7
15:19:08.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.222061 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.61:46833: Network error: Client connection negotiation failed: client connection to 127.28.18.61:46833: connect: Connection refused (error 111)
15:19:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.268750 28859 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:19:08.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.269414 28859 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:19:08.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.270042 28859 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:19:08.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.304491 28859 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:19:08.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.304798 28859 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:19:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.305066 28859 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:19:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.305269 28859 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:19:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.305461 28859 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:19:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.305655 28859 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:19:08.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.305806 28859 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40711/jwks.json
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:08.347568 28859 master_runner.cc:386] Master server non-default flags:
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.61
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.61:46833
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data/info.pb
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.61
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:19:08.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:08.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:19:08.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:19:08.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:08.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/logs
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:08.349086 28859 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:19:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:08.350262 28859 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:19:08.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.362648 28867 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:09.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:09.765560 28866 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 28859
15:19:09.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:09.891223 28859 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.527s user 0.428s sys 1.039s
15:19:09.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:09.891831 28859 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.528s user 0.429s sys 1.039s
15:19:09.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:08.365027 28868 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:09.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:09.893242 28869 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1526 milliseconds
15:19:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:09.893847 28870 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:09.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.893817 28859 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:19:09.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.897269 28859 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:19:09.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.899766 28859 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:19:09.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.901262 28859 hybrid_clock.cc:648] HybridClock initialized: now 1770045549901223 us; error 46 us; skew 500 ppm
15:19:09.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.902169 28859 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:19:09.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:09 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045549, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:09.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.908484 28859 init.cc:377] Logged in from keytab as kudu/127.28.18.61@KRBTEST.COM (short username kudu)
15:19:09.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.913944 28859 webserver.cc:492] Webserver started at http://127.28.18.61:45041/ using document root <none> and password file <none>
15:19:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.915030 28859 fs_manager.cc:362] Metadata directory not provided
15:19:09.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.915303 28859 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:19:09.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.915920 28859 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:19:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.921197 28859 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data/instance:
15:19:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cf4979e8045425d814c265ac16bcb19"
15:19:09.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:09 on dist-test-slave-q1dj"
15:19:09.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.922341 28859 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal/instance:
15:19:09.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cf4979e8045425d814c265ac16bcb19"
15:19:09.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:09 on dist-test-slave-q1dj"
15:19:09.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.931388 28859 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.003s
15:19:09.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.937611 28879 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:19:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.938863 28859 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
15:19:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.939165 28859 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal
15:19:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cf4979e8045425d814c265ac16bcb19"
15:19:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:09 on dist-test-slave-q1dj"
15:19:09.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:09.939477 28859 fs_report.cc:389] FS layout report
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data/data
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:19:09.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:19:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.000599 28859 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:19:10.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.010799 28859 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:19:10.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.011237 28859 kserver.cc:163] Server-wide thread pool size limit: 3276
15:19:10.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.097643 28859 rpc_server.cc:307] RPC server started. Bound to: 127.28.18.61:46833
15:19:10.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.097684 28931 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.18.61:46833 every 8 connection(s)
15:19:10.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.100579 28859 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data/info.pb
15:19:10.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.108225 28932 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:19:10.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.110284 28744 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 28859
15:19:10.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.110755 28744 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal/instance
15:19:10.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.123087 28932 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } has no permanent_uuid. Determining permanent_uuid...
15:19:10.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.18.60@KRBTEST.COM; defaulting to no policy
15:19:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:10.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.18.60@KRBTEST.COM" created.
15:19:10.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:10 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045549, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.61@KRBTEST.COM for kudu/127.28.18.62@KRBTEST.COM
15:19:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:10.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:10.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.166355 28932 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } has no permanent_uuid. Determining permanent_uuid...
15:19:10.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.18.60@KRBTEST.COM; defaulting to no policy
15:19:10.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:10.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.18.60@KRBTEST.COM" created.
15:19:10.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:10 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045549, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.61@KRBTEST.COM for kudu/127.28.18.61@KRBTEST.COM
15:19:10.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.195961 28932 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } has no permanent_uuid. Determining permanent_uuid...
15:19:10.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.198606 28884 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:10.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.201277 28744 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:10.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:10.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal
15:19:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data
15:19:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:19:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:19:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:19:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:10.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/logs
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data/info.pb
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.60:43051
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.60
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.60
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:19:10.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.204510 28932 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:10.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.254106 28932 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 1
15:19:10.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.259212 28932 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:10.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.312598 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } attempt: 8
15:19:10.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:10 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045545, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.62@KRBTEST.COM for kudu/127.28.18.61@KRBTEST.COM
15:19:10.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.327673 28932 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 2
15:19:10.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.332219 28932 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:10.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.341851 28846 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } has no permanent_uuid. Determining permanent_uuid...
15:19:10.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.348743 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:10.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.414258 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 1
15:19:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.419291 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:10.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.423694 28932 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 3
15:19:10.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.428311 28932 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.472786 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 2
15:19:10.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.478482 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:10.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.555126 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 3
15:19:10.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.559722 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:10.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.571764 28932 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 4
15:19:10.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.576686 28932 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:10.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.578647 28942 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:19:10.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.579257 28942 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:19:10.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.579964 28942 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:19:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.613292 28942 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:19:10.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.613554 28942 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:19:10.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.613817 28942 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:19:10.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.614095 28942 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:19:10.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.614424 28942 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:19:10.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.614764 28942 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:19:10.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.615036 28942 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40711/jwks.json
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.656831 28942 master_runner.cc:386] Master server non-default flags:
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.60
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.60:43051
15:19:10.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data/info.pb
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.60
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/logs
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:19:10.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:19:10.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:19:10.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:19:10.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:19:10.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:19:10.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.658337 28942 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:19:10.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.659646 28942 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:19:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.673924 28951 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:10.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.716347 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 4
15:19:10.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.725823 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:10.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:10.876132 28932 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 5
15:19:10.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.883682 28932 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:11.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:11.022471 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 5
15:19:11.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:11.027647 28798 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111) [suppressed 12 similar messages]
15:19:11.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:11.038527 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:11.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:11.431272 28932 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 6
15:19:11.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:11.460448 28932 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:11.501052 28243 debug-util.cc:398] Leaking SignalData structure 0x7b080009b1c0 after lost signal to thread 28173
15:19:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:11.501793 28243 debug-util.cc:398] Leaking SignalData structure 0x7b08000a5d40 after lost signal to thread 28246
15:19:11.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:11.560107 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 6
15:19:11.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:11.618455 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:12.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.521080 28932 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 7
15:19:12.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:12.536413 28932 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:12.076661 28950 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 28942
15:19:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:12.551918 28950 kernel_stack_watchdog.cc:198] Thread 28942 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
15:19:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:19:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:19:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:19:12.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:19:12.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:10.674431 28952 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:12.555248 28942 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.881s user 0.003s sys 0.000s
15:19:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:12.554744 28953 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1878 milliseconds
15:19:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:12.555706 28942 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.882s user 0.003s sys 0.000s
15:19:12.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:12.556928 28954 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:12.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.557277 28942 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:19:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.566126 28942 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:19:12.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.570521 28942 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:19:12.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.572036 28942 hybrid_clock.cc:648] HybridClock initialized: now 1770045552571764 us; error 281 us; skew 500 ppm
15:19:12.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.573277 28942 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:19:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:12 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045552, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.582604 28942 init.cc:377] Logged in from keytab as kudu/127.28.18.60@KRBTEST.COM (short username kudu)
15:19:12.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.589023 28942 webserver.cc:492] Webserver started at http://127.28.18.60:44537/ using document root <none> and password file <none>
15:19:12.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.590191 28942 fs_manager.cc:362] Metadata directory not provided
15:19:12.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.590474 28942 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:19:12.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.591058 28942 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:19:12.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.595968 28942 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data/instance:
15:19:12.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24b594a8de94084a506d2643aeac833"
15:19:12.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:12 on dist-test-slave-q1dj"
15:19:12.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.597467 28942 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal/instance:
15:19:12.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24b594a8de94084a506d2643aeac833"
15:19:12.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:12 on dist-test-slave-q1dj"
15:19:12.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.607316 28942 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.005s
15:19:12.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.613916 28969 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:19:12.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.615284 28942 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
15:19:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.615682 28942 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal
15:19:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24b594a8de94084a506d2643aeac833"
15:19:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:12 on dist-test-slave-q1dj"
15:19:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.616228 28942 fs_report.cc:389] FS layout report
15:19:12.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data/data
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:19:12.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:19:12.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.656049 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 7
15:19:12.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:12.669431 28846 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:12.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.678285 28942 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:19:12.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.686061 28942 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:19:12.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.686673 28942 kserver.cc:163] Server-wide thread pool size limit: 3276
15:19:12.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.799106 29022 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.18.60:43051 every 8 connection(s)
15:19:12.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.799093 28942 rpc_server.cc:307] RPC server started. Bound to: 127.28.18.60:43051
15:19:12.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.803884 28942 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data/info.pb
15:19:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.812625 29023 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:19:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.813750 28744 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 28942
15:19:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.814522 28744 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal/instance
15:19:12.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.830295 29023 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } has no permanent_uuid. Determining permanent_uuid...
15:19:12.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:12 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045543, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.18.62@KRBTEST.COM
15:19:12.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:12 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045552, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.60@KRBTEST.COM for kudu/127.28.18.62@KRBTEST.COM
15:19:12.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.906817 29023 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } has no permanent_uuid. Determining permanent_uuid...
15:19:12.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:12 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045543, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.18.61@KRBTEST.COM
15:19:12.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:12 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045552, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.60@KRBTEST.COM for kudu/127.28.18.61@KRBTEST.COM
15:19:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.941301 29023 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } has no permanent_uuid. Determining permanent_uuid...
15:19:12.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:12 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045543, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.28.18.60@KRBTEST.COM
15:19:12.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:12 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045552, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.60@KRBTEST.COM for kudu/127.28.18.60@KRBTEST.COM
15:19:12.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:12.987043 29023 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: Bootstrap starting.
15:19:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.000213 29023 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: Neither blocks nor log segments found. Creating new log.
15:19:13.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.002712 29023 log.cc:826] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: Log is configured to *not* fsync() on all Append() calls
15:19:13.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.010321 29023 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: No bootstrap required, opened a new log
15:19:13.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.031761 29023 raft_consensus.cc:359] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:13.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.033093 29023 raft_consensus.cc:385] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:19:13.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.033296 29023 raft_consensus.cc:740] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a24b594a8de94084a506d2643aeac833, State: Initialized, Role: FOLLOWER
15:19:13.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.034039 29023 consensus_queue.cc:260] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:13.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.036599 29034 sys_catalog.cc:455] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:13.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.037789 29034 sys_catalog.cc:458] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: This master's current role is: FOLLOWER
15:19:13.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.039613 29023 sys_catalog.cc:565] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: configured and running, proceeding with master startup.
15:19:13.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:13.082211 29045 catalog_manager.cc:1568] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:13.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:13.082939 29045 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:13.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.070664 28942 master_runner.cc:186] Error getting master registration for 127.28.18.62:43993: OK, instance_id { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" instance_seqno: 1770045545735462 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:19:13.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.105381 28942 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:19:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:13.194360 28977 tablet.cc:2378] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:19:13.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.288962 29034 raft_consensus.cc:493] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:19:13.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.289573 29034 raft_consensus.cc:515] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:13.292483 29034 leader_election.cc:290] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 5f40696aca8e42b1bd0548ce27fbc279 (127.28.18.62:43993), 7cf4979e8045425d814c265ac16bcb19 (127.28.18.61:46833)
15:19:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:14.088016 29045 catalog_manager.cc:1568] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:19:14.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:14.088670 29045 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:19:14.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.626930 28932 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 8
15:19:14.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:14 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045549, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.61@KRBTEST.COM for kudu/127.28.18.60@KRBTEST.COM
15:19:14.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.685308 28932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: Bootstrap starting.
15:19:14.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.693563 28932 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: Neither blocks nor log segments found. Creating new log.
15:19:14.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.697316 28932 log.cc:826] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: Log is configured to *not* fsync() on all Append() calls
15:19:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.703179 28932 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: No bootstrap required, opened a new log
15:19:14.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.738479 28932 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:14.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.739387 28932 raft_consensus.cc:385] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:19:14.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.739725 28932 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7cf4979e8045425d814c265ac16bcb19, State: Initialized, Role: FOLLOWER
15:19:14.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.740833 28932 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:14.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.744427 28846 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } attempt: 8
15:19:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.745625 29053 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.746502 29053 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: This master's current role is: FOLLOWER
15:19:14.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.747861 28932 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: configured and running, proceeding with master startup.
15:19:14.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.748050 28907 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a24b594a8de94084a506d2643aeac833" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7cf4979e8045425d814c265ac16bcb19" is_pre_election: true
15:19:14.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.748955 28907 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a24b594a8de94084a506d2643aeac833 in term 0.
15:19:14.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:14.749481 28907 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:51448 (request call id 1) took 1455 ms (client timeout 1681 ms). Trace:
15:19:14.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:14.749744 28907 rpcz_store.cc:269] 0202 15:19:13.293720 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:19:14.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:13.293886 (+ 166us) service_pool.cc:225] Handling call
15:19:14.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:14.749415 (+1455529us) inbound_call.cc:173] Queueing success response
15:19:14.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
15:19:14.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.752177 28973 leader_election.cc:304] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7cf4979e8045425d814c265ac16bcb19, a24b594a8de94084a506d2643aeac833; no voters:
15:19:14.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:14 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045545, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.62@KRBTEST.COM for kudu/127.28.18.60@KRBTEST.COM
15:19:14.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:14.772948 28886 tablet.cc:2378] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:19:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.782258 29056 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:19:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.783330 29056 raft_consensus.cc:493] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:19:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.786010 29056 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 0 FOLLOWER]: Advancing to term 1
15:19:14.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.802722 28846 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: Bootstrap starting.
15:19:14.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.813174 28846 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: Neither blocks nor log segments found. Creating new log.
15:19:14.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.816731 28846 log.cc:826] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: Log is configured to *not* fsync() on all Append() calls
15:19:14.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.829737 28846 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: No bootstrap required, opened a new log
15:19:14.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.828799 29056 raft_consensus.cc:515] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:14.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.841722 28907 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a24b594a8de94084a506d2643aeac833" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7cf4979e8045425d814c265ac16bcb19"
15:19:14.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.842365 28907 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 0 FOLLOWER]: Advancing to term 1
15:19:14.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.847604 29056 leader_election.cc:290] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [CANDIDATE]: Term 1 election: Requested vote from peers 5f40696aca8e42b1bd0548ce27fbc279 (127.28.18.62:43993), 7cf4979e8045425d814c265ac16bcb19 (127.28.18.61:46833)
15:19:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.857932 28907 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a24b594a8de94084a506d2643aeac833 in term 1.
15:19:14.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.860023 28973 leader_election.cc:304] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 7cf4979e8045425d814c265ac16bcb19, a24b594a8de94084a506d2643aeac833; no voters:
15:19:14.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.861128 29056 raft_consensus.cc:2804] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 1 FOLLOWER]: Leader election won for term 1
15:19:14.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.876405 29056 raft_consensus.cc:697] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 1 LEADER]: Becoming Leader. State: Replica: a24b594a8de94084a506d2643aeac833, State: Running, Role: LEADER
15:19:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.877700 29056 consensus_queue.cc:237] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:14.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.884630 28846 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:14.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.886353 28846 raft_consensus.cc:385] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:19:14.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.886953 28846 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5f40696aca8e42b1bd0548ce27fbc279, State: Initialized, Role: FOLLOWER
15:19:14.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.890829 29068 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:19:14.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.889603 28846 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:14.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.906013 29067 sys_catalog.cc:455] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a24b594a8de94084a506d2643aeac833. Latest consensus state: current_term: 1 leader_uuid: "a24b594a8de94084a506d2643aeac833" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:14.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.906816 29067 sys_catalog.cc:458] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: This master's current role is: LEADER
15:19:14.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.916432 29071 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:19:14.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.917865 29070 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.920487 29070 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: This master's current role is: FOLLOWER
15:19:14.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.923547 28820 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a24b594a8de94084a506d2643aeac833" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5f40696aca8e42b1bd0548ce27fbc279"
15:19:14.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.923527 28821 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a24b594a8de94084a506d2643aeac833" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "5f40696aca8e42b1bd0548ce27fbc279" is_pre_election: true
15:19:14.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.924588 29071 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:19:14.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.924520 28820 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 0 FOLLOWER]: Advancing to term 1
15:19:14.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:14.924983 28821 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:43390 (request call id 2) took 1631 ms (client timeout 1681 ms). Trace:
15:19:14.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:14.925410 28821 rpcz_store.cc:269] 0202 15:19:13.293301 (+ 0us) service_pool.cc:168] Inserting onto call queue
15:19:14.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:13.293457 (+ 156us) service_pool.cc:225] Handling call
15:19:14.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:14.924843 (+1631386us) inbound_call.cc:173] Queueing success response
15:19:14.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"spinlock_wait_cycles":1792}
15:19:14.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.927917 28846 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: configured and running, proceeding with master startup.
15:19:14.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.896998 28859 master_runner.cc:186] Error getting master registration for 127.28.18.62:43993: OK, instance_id { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" instance_seqno: 1770045545735462 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:19:14.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.932492 28859 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:19:14.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.935539 28820 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a24b594a8de94084a506d2643aeac833 in term 1.
15:19:14.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:14.942911 28800 tablet.cc:2378] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:19:14.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.956837 28907 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Refusing update from remote peer a24b594a8de94084a506d2643aeac833: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:19:14.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.958936 29067 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:19:14.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.975435 28820 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 1 FOLLOWER]: Refusing update from remote peer a24b594a8de94084a506d2643aeac833: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:19:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.977183 29067 consensus_queue.cc:1048] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:19:14.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.997928 29053 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a24b594a8de94084a506d2643aeac833. Latest consensus state: current_term: 1 leader_uuid: "a24b594a8de94084a506d2643aeac833" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:14.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:14.998989 29053 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: This master's current role is: FOLLOWER
15:19:15.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.006655 29073 mvcc.cc:204] Tried to move back new op lower bound from 7250106593053728768 to 7250106592834506752. Current Snapshot: MvccSnapshot[applied={T|T < 7250106593053728768}]
15:19:15.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.022702 29070 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a24b594a8de94084a506d2643aeac833. Latest consensus state: current_term: 1 leader_uuid: "a24b594a8de94084a506d2643aeac833" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:15.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.023480 29070 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: This master's current role is: FOLLOWER
15:19:15.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.032713 29067 sys_catalog.cc:455] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a24b594a8de94084a506d2643aeac833" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:15.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.033746 29067 sys_catalog.cc:458] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: This master's current role is: LEADER
15:19:15.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.034965 29056 sys_catalog.cc:455] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a24b594a8de94084a506d2643aeac833" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.035913 29056 sys_catalog.cc:458] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: This master's current role is: LEADER
15:19:15.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.035894 29070 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a24b594a8de94084a506d2643aeac833" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.036613 29070 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: This master's current role is: FOLLOWER
15:19:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.037489 29053 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a24b594a8de94084a506d2643aeac833" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:15.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.038187 29053 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: This master's current role is: FOLLOWER
15:19:15.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.042028 29071 catalog_manager.cc:1357] Generated new cluster ID: f403ce6f995c46b58cc12ed370c76892
15:19:15.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.042336 29071 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:19:15.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.062637 29090 catalog_manager.cc:1269] Loaded cluster ID: f403ce6f995c46b58cc12ed370c76892
15:19:15.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.062992 29090 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: loading cluster ID for follower catalog manager: success
15:19:15.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:15.068207 29090 catalog_manager.cc:1586] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: acquiring CA information for follower catalog manager: Not found: root CA entry not found
15:19:15.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:15.068593 29090 catalog_manager.cc:883] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
15:19:15.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.121619 29071 catalog_manager.cc:1380] Generated new certificate authority record
15:19:15.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.123759 29071 catalog_manager.cc:1514] Loading token signing keys...
15:19:15.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.167193 29071 catalog_manager.cc:6027] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: Generated new TSK 0
15:19:15.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.168442 29071 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:19:15.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.18.1@KRBTEST.COM; defaulting to no policy
15:19:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:15.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.18.1@KRBTEST.COM" created.
15:19:15.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:15.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:15.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.18.1@KRBTEST.COM; defaulting to no policy
15:19:15.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:15.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.18.1@KRBTEST.COM" created.
15:19:15.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:15.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:15.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.284075 28744 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/wal
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/data
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/logs
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/data/info.pb
15:19:15.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.1:0
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.18.1
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.1
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.1
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:15.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:15.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:19:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:15 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045555, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:15.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.579434 28790 init.cc:295] Successfully reacquired a new kerberos TGT
15:19:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:15.723961 29096 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:19:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:15.724545 29096 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:19:15.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:15.725524 29096 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:19:15.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:15.767643 29096 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:19:15.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:15.769279 29096 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.18.1
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.868781 29096 tablet_server_runner.cc:78] Tablet server non-default flags:
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/data
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/wal
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.1
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.1:0
15:19:15.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/data/info.pb
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.1
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.18.1
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/logs
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:19:15.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:19:15.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:19:15.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.870575 29096 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:19:15.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.872078 29096 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:19:15.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.920035 29068 catalog_manager.cc:1269] Loaded cluster ID: f403ce6f995c46b58cc12ed370c76892
15:19:15.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.921672 29068 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: loading cluster ID for follower catalog manager: success
15:19:15.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:15.927912 29105 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:15.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.935452 29068 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: acquiring CA information for follower catalog manager: success
15:19:15.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:15.957458 29068 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:19:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:16.077090 29090 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: acquiring CA information for follower catalog manager: success
15:19:16.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:16.091661 29090 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:19:17.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:15.930778 29106 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:17.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:17.308944 29102 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 29096
15:19:17.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:17.786170 29096 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.864s user 0.627s sys 1.128s
15:19:17.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:17.788137 29096 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.867s user 0.627s sys 1.128s
15:19:17.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:17.808745 29115 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:17.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.818645 29096 server_base.cc:1060] running on GCE node
15:19:17.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.820142 29096 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:19:17.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.835420 29096 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:19:17.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.837286 29096 hybrid_clock.cc:648] HybridClock initialized: now 1770045557837232 us; error 57 us; skew 500 ppm
15:19:17.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.838497 29096 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:19:17.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:17 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045557, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:17.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.847637 29096 init.cc:377] Logged in from keytab as kudu/127.28.18.1@KRBTEST.COM (short username kudu)
15:19:17.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.855170 29096 webserver.cc:492] Webserver started at http://127.28.18.1:33497/ using document root <none> and password file <none>
15:19:17.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.856542 29096 fs_manager.cc:362] Metadata directory not provided
15:19:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.856880 29096 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:19:17.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.857522 29096 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:19:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.864109 29096 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/data/instance:
15:19:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea7cb7963e304e4b8938a5f263dd8621"
15:19:17.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:17 on dist-test-slave-q1dj"
15:19:17.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.865787 29096 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/wal/instance:
15:19:17.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea7cb7963e304e4b8938a5f263dd8621"
15:19:17.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:17 on dist-test-slave-q1dj"
15:19:17.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.877133 29096 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.007s sys 0.004s
15:19:17.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.885982 29121 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:19:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.887668 29096 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.002s sys 0.004s
15:19:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.888130 29096 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/wal
15:19:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ea7cb7963e304e4b8938a5f263dd8621"
15:19:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:17 on dist-test-slave-q1dj"
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.888584 29096 fs_report.cc:389] FS layout report
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/wal
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/wal
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/data/data
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:19:17.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:19:17.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.968509 29096 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:19:17.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.976441 29096 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:19:17.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.977094 29096 kserver.cc:163] Server-wide thread pool size limit: 3276
15:19:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.986474 29096 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:19:17.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.999572 29096 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:19:18.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:17.999922 29096 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
15:19:18.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.000239 29096 ts_tablet_manager.cc:616] Registered 0 tablets
15:19:18.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.000452 29096 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:19:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.365875 29096 rpc_server.cc:307] RPC server started. Bound to: 127.28.18.1:44473
15:19:18.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.366034 29234 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.18.1:44473 every 8 connection(s)
15:19:18.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.369714 29096 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/data/info.pb
15:19:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.375965 28744 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 29096
15:19:18.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.376449 28744 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-0/wal/instance
15:19:18.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.18.2@KRBTEST.COM; defaulting to no policy
15:19:18.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:18 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045557, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.1@KRBTEST.COM for kudu/127.28.18.60@KRBTEST.COM
15:19:18.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:18 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045557, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.1@KRBTEST.COM for kudu/127.28.18.61@KRBTEST.COM
15:19:18.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:18.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.18.2@KRBTEST.COM" created.
15:19:18.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:18 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045557, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.1@KRBTEST.COM for kudu/127.28.18.62@KRBTEST.COM
15:19:18.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.519304 29236 heartbeater.cc:344] Connected to a master server at 127.28.18.61:46833
15:19:18.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.519897 29236 heartbeater.cc:461] Registering TS with master...
15:19:18.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.521346 29236 heartbeater.cc:507] Master 127.28.18.61:46833 requested a full tablet report, sending...
15:19:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.522970 29235 heartbeater.cc:344] Connected to a master server at 127.28.18.60:43051
15:19:18.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.523469 29235 heartbeater.cc:461] Registering TS with master...
15:19:18.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.524637 29235 heartbeater.cc:507] Master 127.28.18.60:43051 requested a full tablet report, sending...
15:19:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.525945 28897 ts_manager.cc:194] Registered new tserver with Master: ea7cb7963e304e4b8938a5f263dd8621 (127.28.18.1:44473)
15:19:18.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.528200 28988 ts_manager.cc:194] Registered new tserver with Master: ea7cb7963e304e4b8938a5f263dd8621 (127.28.18.1:44473)
15:19:18.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.533195 28988 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.18.1@KRBTEST.COM'} at 127.28.18.1:40265
15:19:18.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.557744 29239 heartbeater.cc:344] Connected to a master server at 127.28.18.62:43993
15:19:18.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.558184 29239 heartbeater.cc:461] Registering TS with master...
15:19:18.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.558923 29239 heartbeater.cc:507] Master 127.28.18.62:43993 requested a full tablet report, sending...
15:19:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.562482 28811 ts_manager.cc:194] Registered new tserver with Master: ea7cb7963e304e4b8938a5f263dd8621 (127.28.18.1:44473)
15:19:18.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:18.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:18.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:18.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:18.568687 28744 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.28.18.2: real 0.167s user 0.004s sys 0.010s
15:19:18.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.18.2@KRBTEST.COM; defaulting to no policy
15:19:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:18.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.18.2@KRBTEST.COM" created.
15:19:18.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:18.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:18.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:18.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:18.624706 28744 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/wal
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/data
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/logs
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/data/info.pb
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.2:0
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.18.2
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.2
15:19:18.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.2
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:18.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:19:19.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:19.080686 29250 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:19:19.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:19.081250 29250 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:19:19.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:19.081995 29250 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:19:19.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:19.133797 29250 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:19:19.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:19.135382 29250 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.18.2
15:19:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:19.189141 29250 tablet_server_runner.cc:78] Tablet server non-default flags:
15:19:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/data
15:19:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/wal
15:19:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.2
15:19:19.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.2:0
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/data/info.pb
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.2
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:19.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.18.2
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/logs
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:19.190423 29250 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:19:19.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:19.191427 29250 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:19:19.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:19.211388 29256 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:19.546613 29235 heartbeater.cc:499] Master 127.28.18.60:43051 was elected leader, sending a full tablet report...
15:19:19.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:19 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045559, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:19.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:19.914867 28875 init.cc:295] Successfully reacquired a new kerberos TGT
15:19:21.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:20.613081 29255 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 29250
15:19:21.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:21.185875 29255 kernel_stack_watchdog.cc:198] Thread 29250 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
15:19:21.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:19:21.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:19:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:19:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:19:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:21.187153 29258 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1979 milliseconds
15:19:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:21.187608 29250 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.980s user 0.747s sys 1.211s
15:19:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:21.188002 29250 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.981s user 0.747s sys 1.211s
15:19:21.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:19.214176 29257 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:21.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.189615 29250 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:19:21.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:21.189658 29259 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:21.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.194005 29250 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:19:21.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.196424 29250 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:19:21.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.197940 29250 hybrid_clock.cc:648] HybridClock initialized: now 1770045561197889 us; error 59 us; skew 500 ppm
15:19:21.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.198835 29250 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:19:21.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:21 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045561, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:21.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.205338 29250 init.cc:377] Logged in from keytab as kudu/127.28.18.2@KRBTEST.COM (short username kudu)
15:19:21.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.210886 29250 webserver.cc:492] Webserver started at http://127.28.18.2:45969/ using document root <none> and password file <none>
15:19:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.211977 29250 fs_manager.cc:362] Metadata directory not provided
15:19:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.212231 29250 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:19:21.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.212767 29250 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:19:21.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.217442 29250 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/data/instance:
15:19:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b65e8d3b11c04b48885d80fad2fc9f6a"
15:19:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:21 on dist-test-slave-q1dj"
15:19:21.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.218639 29250 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/wal/instance:
15:19:21.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b65e8d3b11c04b48885d80fad2fc9f6a"
15:19:21.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:21 on dist-test-slave-q1dj"
15:19:21.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.227283 29250 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.003s
15:19:21.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.233422 29272 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:19:21.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.234624 29250 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:19:21.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.235035 29250 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/wal
15:19:21.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b65e8d3b11c04b48885d80fad2fc9f6a"
15:19:21.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:21 on dist-test-slave-q1dj"
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.235453 29250 fs_report.cc:389] FS layout report
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/wal
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/wal
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/data/data
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:19:21.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:19:21.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.288805 29250 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:19:21.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.298060 29250 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:19:21.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.298543 29250 kserver.cc:163] Server-wide thread pool size limit: 3276
15:19:21.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.301195 29250 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:19:21.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.305917 29250 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:19:21.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.306247 29250 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:19:21.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.306471 29250 ts_tablet_manager.cc:616] Registered 0 tablets
15:19:21.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.306613 29250 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:19:21.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.486644 29250 rpc_server.cc:307] RPC server started. Bound to: 127.28.18.2:45353
15:19:21.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.486805 29384 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.18.2:45353 every 8 connection(s)
15:19:21.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.489387 29250 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/data/info.pb
15:19:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.495986 28744 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 29250
15:19:21.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.496708 28744 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-1/wal/instance
15:19:21.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:21 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045561, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.2@KRBTEST.COM for kudu/127.28.18.61@KRBTEST.COM
15:19:21.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:21 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045561, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.2@KRBTEST.COM for kudu/127.28.18.60@KRBTEST.COM
15:19:21.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:21 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045561, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.2@KRBTEST.COM for kudu/127.28.18.62@KRBTEST.COM
15:19:21.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.28.18.3@KRBTEST.COM; defaulting to no policy
15:19:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:21.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.28.18.3@KRBTEST.COM" created.
15:19:21.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.561245 29386 heartbeater.cc:344] Connected to a master server at 127.28.18.61:46833
15:19:21.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.561836 29386 heartbeater.cc:461] Registering TS with master...
15:19:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.563750 29386 heartbeater.cc:507] Master 127.28.18.61:46833 requested a full tablet report, sending...
15:19:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.564893 29385 heartbeater.cc:344] Connected to a master server at 127.28.18.60:43051
15:19:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.565237 29385 heartbeater.cc:461] Registering TS with master...
15:19:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.566226 29385 heartbeater.cc:507] Master 127.28.18.60:43051 requested a full tablet report, sending...
15:19:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.567329 28897 ts_manager.cc:194] Registered new tserver with Master: b65e8d3b11c04b48885d80fad2fc9f6a (127.28.18.2:45353)
15:19:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.568866 28988 ts_manager.cc:194] Registered new tserver with Master: b65e8d3b11c04b48885d80fad2fc9f6a (127.28.18.2:45353)
15:19:21.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.570909 28988 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.18.2@KRBTEST.COM'} at 127.28.18.2:38897
15:19:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.579744 29387 heartbeater.cc:344] Connected to a master server at 127.28.18.62:43993
15:19:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.580206 29387 heartbeater.cc:461] Registering TS with master...
15:19:21.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.580960 29387 heartbeater.cc:507] Master 127.28.18.62:43993 requested a full tablet report, sending...
15:19:21.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.583935 28811 ts_manager.cc:194] Registered new tserver with Master: b65e8d3b11c04b48885d80fad2fc9f6a (127.28.18.2:45353)
15:19:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:21.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.28.18.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:21.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.28.18.3@KRBTEST.COM; defaulting to no policy
15:19:21.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:21.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.28.18.3@KRBTEST.COM" created.
15:19:21.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:19:21.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:21.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.28.18.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab.
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:21.636072 28744 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/wal
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/data
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/logs
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/data/info.pb
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.3:0
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.18.3
15:19:21.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.3
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.3
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:21.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:19:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:21.994589 29399 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:19:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:21.995100 29399 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:19:21.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:21.995723 29399 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:19:22.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:22.035898 29399 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:19:22.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:22.037230 29399 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.18.3
15:19:22.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:22.094358 29399 tablet_server_runner.cc:78] Tablet server non-default flags:
15:19:22.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:22.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:22.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:22.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:22.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/data
15:19:22.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/wal
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.3
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.3:0
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/data/info.pb
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.3
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.18.3
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/logs
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:19:22.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:19:22.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:22.095762 29399 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:19:22.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:22.096967 29399 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:19:22.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:22.111474 29406 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:22.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:22.575933 29385 heartbeater.cc:499] Master 127.28.18.60:43051 was elected leader, sending a full tablet report...
15:19:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:22 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045562, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:22.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:22.603673 28965 init.cc:295] Successfully reacquired a new kerberos TGT
15:19:23.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:22.114290 29407 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:23.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:23.528936 29405 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 29399
15:19:23.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:23.765035 29405 kernel_stack_watchdog.cc:198] Thread 29399 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 410ms:
15:19:23.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:19:23.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:19:23.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:19:23.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:19:23.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:23.765440 29399 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.650s user 0.000s sys 0.002s
15:19:23.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:23.765837 29399 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.651s user 0.000s sys 0.002s
15:19:23.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:23.768620 29408 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1653 milliseconds
15:19:23.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.768858 29399 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:19:23.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:23.769112 29409 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:23.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.774513 29399 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:19:23.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.777168 29399 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:19:23.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.778680 29399 hybrid_clock.cc:648] HybridClock initialized: now 1770045563778633 us; error 49 us; skew 500 ppm
15:19:23.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.779796 29399 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:19:23.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:23 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045563, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:23.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.786715 29399 init.cc:377] Logged in from keytab as kudu/127.28.18.3@KRBTEST.COM (short username kudu)
15:19:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.792737 29399 webserver.cc:492] Webserver started at http://127.28.18.3:40075/ using document root <none> and password file <none>
15:19:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.793800 29399 fs_manager.cc:362] Metadata directory not provided
15:19:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.794070 29399 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:19:23.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.794624 29399 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
15:19:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.800597 29399 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/data/instance:
15:19:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3e065f11b1814304b8944abe3a89961d"
15:19:23.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:23 on dist-test-slave-q1dj"
15:19:23.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.802484 29399 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/wal/instance:
15:19:23.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3e065f11b1814304b8944abe3a89961d"
15:19:23.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:23 on dist-test-slave-q1dj"
15:19:23.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.812299 29399 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.007s sys 0.000s
15:19:23.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.818625 29422 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:19:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.820001 29399 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
15:19:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.820483 29399 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/wal
15:19:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3e065f11b1814304b8944abe3a89961d"
15:19:23.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:23 on dist-test-slave-q1dj"
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.820814 29399 fs_report.cc:389] FS layout report
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/wal
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/wal
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/data/data
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:19:23.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:19:23.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.876780 29399 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:19:23.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.883205 29399 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:19:23.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.883782 29399 kserver.cc:163] Server-wide thread pool size limit: 3276
15:19:23.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.886569 29399 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:19:23.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.890892 29399 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:19:23.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.891193 29399 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:19:23.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.891462 29399 ts_tablet_manager.cc:616] Registered 0 tablets
15:19:23.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:23.891605 29399 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:19:24.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.065062 29399 rpc_server.cc:307] RPC server started. Bound to: 127.28.18.3:34349
15:19:24.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.065229 29534 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.18.3:34349 every 8 connection(s)
15:19:24.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.068235 29399 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/data/info.pb
15:19:24.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.078084 28744 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 29399
15:19:24.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.078805 28744 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/ts-2/wal/instance
15:19:24.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:24 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045563, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.3@KRBTEST.COM for kudu/127.28.18.60@KRBTEST.COM
15:19:24.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:24 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045563, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.3@KRBTEST.COM for kudu/127.28.18.61@KRBTEST.COM
15:19:24.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:24 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045563, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.3@KRBTEST.COM for kudu/127.28.18.62@KRBTEST.COM
15:19:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.173661 29535 heartbeater.cc:344] Connected to a master server at 127.28.18.60:43051
15:19:24.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.174216 29535 heartbeater.cc:461] Registering TS with master...
15:19:24.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.175577 29535 heartbeater.cc:507] Master 127.28.18.60:43051 requested a full tablet report, sending...
15:19:24.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.178668 28988 ts_manager.cc:194] Registered new tserver with Master: 3e065f11b1814304b8944abe3a89961d (127.28.18.3:34349)
15:19:24.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.180464 28988 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.28.18.3@KRBTEST.COM'} at 127.28.18.3:48739
15:19:24.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.182471 29536 heartbeater.cc:344] Connected to a master server at 127.28.18.61:46833
15:19:24.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.182873 29536 heartbeater.cc:461] Registering TS with master...
15:19:24.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.184782 29536 heartbeater.cc:507] Master 127.28.18.61:46833 requested a full tablet report, sending...
15:19:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.187671 28897 ts_manager.cc:194] Registered new tserver with Master: 3e065f11b1814304b8944abe3a89961d (127.28.18.3:34349)
15:19:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.189230 29537 heartbeater.cc:344] Connected to a master server at 127.28.18.62:43993
15:19:24.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.189633 29537 heartbeater.cc:461] Registering TS with master...
15:19:24.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.190318 29537 heartbeater.cc:507] Master 127.28.18.62:43993 requested a full tablet report, sending...
15:19:24.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.193717 28811 ts_manager.cc:194] Registered new tserver with Master: 3e065f11b1814304b8944abe3a89961d (127.28.18.3:34349)
15:19:24.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.197304 28744 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
15:19:24.212 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:24.213 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:19:24.224 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:19:24.225 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:19:24.228 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.18.62"
port: 43993
}
bound_http_address {
host: "127.28.18.62"
port: 36881
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.18.61"
port: 46833
}
bound_http_address {
host: "127.28.18.61"
port: 45041
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.18.60"
port: 43051
}
bound_http_address {
host: "127.28.18.60"
port: 44537
}
}
}
15:19:24.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:19:24.230 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.28.18.1"
port: 44473
}
bound_http_address {
host: "127.28.18.1"
port: 33497
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.28.18.2"
port: 45353
}
bound_http_address {
host: "127.28.18.2"
port: 45969
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.28.18.3"
port: 34349
}
bound_http_address {
host: "127.28.18.3"
port: 40075
}
}
}
15:19:24.230 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:19:24.231 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:19:24.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:24 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045564, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:24.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:19:24.265 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:24.267 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc
15:19:24.273 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:19:24.274 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc
15:19:24.275 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:19:24.278 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
15:19:24.280 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:19:24.281 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:19:24.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:24 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045564, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:24.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:19:24.313 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:24.314 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc
15:19:24.315 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:19:24.316 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.18.61:46833
15:19:24.317 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:19:24.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.318492 28744 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 28859
15:19:24.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:24.346 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.18.62:43993
15:19:24.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:19:24.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.347271 28744 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 28774
15:19:24.384 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:24.384 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.28.18.60:43051
15:19:24.384 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:19:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.384881 28744 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 28942
15:19:24.413 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:24.413 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.18.61:46833
15:19:24.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:19:24.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.419150 28744 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:24.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/logs
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data/info.pb
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.61:46833
15:19:24.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.61
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45041
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.61
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:19:24.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:19:24.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.557991 29236 heartbeater.cc:646] Failed to heartbeat to 127.28.18.61:46833 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.18.61:46833: connect: Connection refused (error 111)
15:19:24.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.583415 29386 heartbeater.cc:646] Failed to heartbeat to 127.28.18.61:46833 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.18.61:46833: connect: Connection refused (error 111)
15:19:24.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.753365 29549 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:19:24.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.753942 29549 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:19:24.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.754578 29549 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:19:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.792271 29549 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:19:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.792518 29549 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:19:24.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.792776 29549 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:19:24.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.792969 29549 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:19:24.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.793185 29549 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:19:24.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.793380 29549 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:19:24.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.793541 29549 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40711/jwks.json
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.833870 29549 master_runner.cc:386] Master server non-default flags:
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.61
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.61:46833
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data/info.pb
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.61
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45041
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/logs
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:19:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.835162 29549 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:19:24.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:24.836382 29549 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:19:24.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.849190 29558 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:25.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:25.186033 29535 heartbeater.cc:499] Master 127.28.18.60:43051 was elected leader, sending a full tablet report...
15:19:25.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:25.212354 29535 heartbeater.cc:646] Failed to heartbeat to 127.28.18.60:43051 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:26.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:26.034247 29381 debug-util.cc:398] Leaking SignalData structure 0x7b08000bb600 after lost signal to thread 29251
15:19:26.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:26.040556 29381 debug-util.cc:398] Leaking SignalData structure 0x7b08000bbf40 after lost signal to thread 29384
15:19:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:26.256371 29557 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 29549
15:19:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:24.850155 29559 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:26.742233 29549 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.893s user 0.607s sys 1.218s
15:19:26.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:26.742666 29549 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.894s user 0.607s sys 1.218s
15:19:26.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:26.744835 29561 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:26.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:26.750105 29560 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1900 milliseconds
15:19:26.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.750169 29549 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:19:26.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.751955 29549 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:19:26.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.757423 29549 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:19:26.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.759299 29549 hybrid_clock.cc:648] HybridClock initialized: now 1770045566759192 us; error 99 us; skew 500 ppm
15:19:26.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.760571 29549 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:19:26.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:26 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045566, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:26.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.767797 29549 init.cc:377] Logged in from keytab as kudu/127.28.18.61@KRBTEST.COM (short username kudu)
15:19:26.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.774806 29549 webserver.cc:492] Webserver started at http://127.28.18.61:45041/ using document root <none> and password file <none>
15:19:26.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.776147 29549 fs_manager.cc:362] Metadata directory not provided
15:19:26.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.776928 29549 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:19:26.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.788712 29549 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.004s sys 0.004s
15:19:26.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.794401 29584 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:19:26.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.795985 29549 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
15:19:26.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.796451 29549 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal
15:19:26.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7cf4979e8045425d814c265ac16bcb19"
15:19:26.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:09 on dist-test-slave-q1dj"
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.799700 29549 fs_report.cc:389] FS layout report
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/wal
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data/data
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:19:26.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:19:26.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.876343 29549 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:19:26.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.890070 29549 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:19:26.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:26.890635 29549 kserver.cc:163] Server-wide thread pool size limit: 3276
15:19:27.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.047688 29549 rpc_server.cc:307] RPC server started. Bound to: 127.28.18.61:46833
15:19:27.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.047891 29635 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.18.61:46833 every 8 connection(s)
15:19:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.051362 29549 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-1/data/info.pb
15:19:27.064 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:27.064 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.18.62:43993
15:19:27.064 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:19:27.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.063159 28744 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 29549
15:19:27.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.066437 28744 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:27.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/logs
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data/info.pb
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:27.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:19:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:19:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.62:43993
15:19:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.62
15:19:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36881
15:19:27.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.62
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:19:27.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.068835 29636 sys_catalog.cc:263] Verifying existing consensus state
15:19:27.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.077749 29636 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: Bootstrap starting.
15:19:27.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.138263 29636 log.cc:826] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: Log is configured to *not* fsync() on all Append() calls
15:19:27.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.161002 29636 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:19:27.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.162110 29636 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: Bootstrap complete.
15:19:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.200570 29636 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:27.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.204231 29636 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7cf4979e8045425d814c265ac16bcb19, State: Initialized, Role: FOLLOWER
15:19:27.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.205380 29636 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.229105 29640 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.233758 29640 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: This master's current role is: FOLLOWER
15:19:27.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.231436 29636 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: configured and running, proceeding with master startup.
15:19:27.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.305212 29536 heartbeater.cc:344] Connected to a master server at 127.28.18.61:46833
15:19:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.350623 29654 catalog_manager.cc:1269] Loaded cluster ID: f403ce6f995c46b58cc12ed370c76892
15:19:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.351305 29654 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: loading cluster ID for follower catalog manager: success
15:19:27.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.359498 29654 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: acquiring CA information for follower catalog manager: success
15:19:27.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.365976 29654 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:19:27.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.351239 29549 master_runner.cc:186] Error getting master registration for 127.28.18.62:43993: Network error: Client connection negotiation failed: client connection to 127.28.18.62:43993: connect: Connection refused (error 111),
15:19:27.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.378352 29549 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:19:27.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.543138 29638 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:19:27.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.543915 29638 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:19:27.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.544721 29638 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:19:27.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.590454 29638 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:19:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.590844 29638 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:19:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.591415 29638 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:19:27.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.591898 29638 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:19:27.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.592343 29638 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:19:27.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.592788 29638 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:19:27.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.593134 29638 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40711/jwks.json
15:19:27.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.619688 29236 heartbeater.cc:344] Connected to a master server at 127.28.18.61:46833
15:19:27.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.633773 29601 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ea7cb7963e304e4b8938a5f263dd8621" instance_seqno: 1770045558300890) as {username='kudu', principal='kudu/127.28.18.1@KRBTEST.COM'} at 127.28.18.1:41935; Asking this server to re-register.
15:19:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.680966 29638 master_runner.cc:386] Master server non-default flags:
15:19:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data
15:19:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal
15:19:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:19:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:27.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:27.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.62
15:19:27.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:27.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:27.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:27.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.62:43993
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data/info.pb
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.62
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36881
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:19:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/logs
15:19:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:19:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:19:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:19:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:19:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:19:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:19:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:19:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:19:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:19:27.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.682745 29638 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:19:27.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.695081 29638 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:19:27.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.699260 29236 heartbeater.cc:461] Registering TS with master...
15:19:27.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.703099 29601 ts_manager.cc:194] Registered new tserver with Master: ea7cb7963e304e4b8938a5f263dd8621 (127.28.18.1:44473)
15:19:27.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.711761 29386 heartbeater.cc:344] Connected to a master server at 127.28.18.61:46833
15:19:27.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.715251 29601 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b65e8d3b11c04b48885d80fad2fc9f6a" instance_seqno: 1770045561446447) as {username='kudu', principal='kudu/127.28.18.2@KRBTEST.COM'} at 127.28.18.2:59563; Asking this server to re-register.
15:19:27.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.717813 29386 heartbeater.cc:461] Registering TS with master...
15:19:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.721387 29601 ts_manager.cc:194] Registered new tserver with Master: b65e8d3b11c04b48885d80fad2fc9f6a (127.28.18.2:45353)
15:19:27.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.724272 29667 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:27.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:27 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045567, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:27.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:27.854223 29117 init.cc:295] Successfully reacquired a new kerberos TGT
15:19:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:28.312539 29601 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "3e065f11b1814304b8944abe3a89961d" instance_seqno: 1770045564023084) as {username='kudu', principal='kudu/127.28.18.3@KRBTEST.COM'} at 127.28.18.3:57203; Asking this server to re-register.
15:19:28.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:28.324115 29536 heartbeater.cc:461] Registering TS with master...
15:19:28.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:28.327464 29601 ts_manager.cc:194] Registered new tserver with Master: 3e065f11b1814304b8944abe3a89961d (127.28.18.3:34349)
15:19:29.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:28.999788 29680 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:19:29.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:29.000890 29680 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:29.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:29.027252 29680 leader_election.cc:290] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5f40696aca8e42b1bd0548ce27fbc279 (127.28.18.62:43993), a24b594a8de94084a506d2643aeac833 (127.28.18.60:43051)
15:19:29.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:29.038959 29588 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:29.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:29.050824 29587 leader_election.cc:336] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 5f40696aca8e42b1bd0548ce27fbc279 (127.28.18.62:43993): Network error: Client connection negotiation failed: client connection to 127.28.18.62:43993: connect: Connection refused (error 111)
15:19:29.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:29.055294 29588 leader_election.cc:336] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a24b594a8de94084a506d2643aeac833 (127.28.18.60:43051): Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:29.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:29.056149 29588 leader_election.cc:304] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 7cf4979e8045425d814c265ac16bcb19; no voters: 5f40696aca8e42b1bd0548ce27fbc279, a24b594a8de94084a506d2643aeac833
15:19:29.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:29.057228 29680 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
15:19:30.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:29.116868 29666 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 29638
15:19:30.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:30.410699 29666 kernel_stack_watchdog.cc:198] Thread 29638 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 389ms:
15:19:30.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:19:30.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:19:30.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:19:30.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:19:30.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:30.411655 29638 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.686s user 0.528s sys 0.843s
15:19:30.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:30.412122 29638 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.687s user 0.528s sys 0.843s
15:19:30.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:27.726876 29668 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:30.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:30.413228 29669 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2686 milliseconds
15:19:30.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.413779 29638 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:19:30.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:30.413837 29671 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.419137 29638 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:19:30.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.421773 29638 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:19:30.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.425339 29638 hybrid_clock.cc:648] HybridClock initialized: now 1770045570425295 us; error 55 us; skew 500 ppm
15:19:30.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.426312 29638 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:19:30.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:30 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045570, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.432830 29638 init.cc:377] Logged in from keytab as kudu/127.28.18.62@KRBTEST.COM (short username kudu)
15:19:30.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.440959 29638 webserver.cc:492] Webserver started at http://127.28.18.62:36881/ using document root <none> and password file <none>
15:19:30.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.441980 29638 fs_manager.cc:362] Metadata directory not provided
15:19:30.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.442260 29638 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:19:30.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.451635 29638 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
15:19:30.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.457008 29698 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:19:30.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.458539 29638 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.005s
15:19:30.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.458963 29638 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal
15:19:30.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "5f40696aca8e42b1bd0548ce27fbc279"
15:19:30.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:05 on dist-test-slave-q1dj"
15:19:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.461046 29638 fs_report.cc:389] FS layout report
15:19:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:19:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal
15:19:30.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/wal
15:19:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data/data
15:19:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:19:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:19:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:19:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:19:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:19:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:19:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:19:30.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:19:30.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:19:30.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:19:30.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:19:30.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.562152 29638 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:19:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.569787 29638 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:19:30.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.570353 29638 kserver.cc:163] Server-wide thread pool size limit: 3276
15:19:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.681681 29638 rpc_server.cc:307] RPC server started. Bound to: 127.28.18.62:43993
15:19:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.681833 29750 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.18.62:43993 every 8 connection(s)
15:19:30.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.684684 29638 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-0/data/info.pb
15:19:30.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.686872 29751 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:19:30.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.687520 29751 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:30.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.694084 28744 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 29638
15:19:30.695 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:30.696 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.28.18.60:43051
15:19:30.696 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.702025 28744 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/logs
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data/info.pb
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.60:43051
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.60
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44537
15:19:30.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.60
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:19:30.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:19:30.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.718505 29752 sys_catalog.cc:263] Verifying existing consensus state
15:19:30.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.721948 29751 leader_election.cc:290] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 5f40696aca8e42b1bd0548ce27fbc279 (127.28.18.62:43993), a24b594a8de94084a506d2643aeac833 (127.28.18.60:43051)
15:19:30.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.729700 29752 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: Bootstrap starting.
15:19:30.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:30 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045566, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.61@KRBTEST.COM for kudu/127.28.18.62@KRBTEST.COM
15:19:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:30.765630 29588 leader_election.cc:336] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a24b594a8de94084a506d2643aeac833 (127.28.18.60:43051): Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:30.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.844786 29752 log.cc:826] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: Log is configured to *not* fsync() on all Append() calls
15:19:30.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:30 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045567, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.1@KRBTEST.COM for kudu/127.28.18.62@KRBTEST.COM
15:19:30.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.869773 29239 heartbeater.cc:344] Connected to a master server at 127.28.18.62:43993
15:19:30.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.892905 29387 heartbeater.cc:344] Connected to a master server at 127.28.18.62:43993
15:19:30.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.914278 29752 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:19:30.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.915766 29752 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: Bootstrap complete.
15:19:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.945844 29752 raft_consensus.cc:359] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:30.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.949384 29752 raft_consensus.cc:740] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5f40696aca8e42b1bd0548ce27fbc279, State: Initialized, Role: FOLLOWER
15:19:30.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.950424 29752 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:30.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.954355 29765 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:30.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.957518 29765 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: This master's current role is: FOLLOWER
15:19:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.962042 29726 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7cf4979e8045425d814c265ac16bcb19" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5f40696aca8e42b1bd0548ce27fbc279" is_pre_election: true
15:19:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.963003 29726 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7cf4979e8045425d814c265ac16bcb19 in term 1.
15:19:30.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.964636 29587 leader_election.cc:304] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5f40696aca8e42b1bd0548ce27fbc279, 7cf4979e8045425d814c265ac16bcb19; no voters: a24b594a8de94084a506d2643aeac833
15:19:30.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.965927 29751 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Leader pre-election won for term 2
15:19:30.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.966286 29751 raft_consensus.cc:493] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:19:30.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.966568 29751 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 1 FOLLOWER]: Advancing to term 2
15:19:30.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.977896 29752 sys_catalog.cc:565] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: configured and running, proceeding with master startup.
15:19:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.981096 29751 raft_consensus.cc:515] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:30.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.987165 29726 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "7cf4979e8045425d814c265ac16bcb19" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "5f40696aca8e42b1bd0548ce27fbc279"
15:19:30.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.987766 29726 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 1 FOLLOWER]: Advancing to term 2
15:19:30.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:30.987942 29751 leader_election.cc:290] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [CANDIDATE]: Term 2 election: Requested vote from peers 5f40696aca8e42b1bd0548ce27fbc279 (127.28.18.62:43993), a24b594a8de94084a506d2643aeac833 (127.28.18.60:43051)
15:19:30.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:30.992529 29588 leader_election.cc:336] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer a24b594a8de94084a506d2643aeac833 (127.28.18.60:43051): Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111)
15:19:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.004113 29726 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7cf4979e8045425d814c265ac16bcb19 in term 2.
15:19:31.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.005323 29587 leader_election.cc:304] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 5f40696aca8e42b1bd0548ce27fbc279, 7cf4979e8045425d814c265ac16bcb19; no voters: a24b594a8de94084a506d2643aeac833
15:19:31.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.006536 29751 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 2 FOLLOWER]: Leader election won for term 2
15:19:31.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.015362 29751 raft_consensus.cc:697] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [term 2 LEADER]: Becoming Leader. State: Replica: 7cf4979e8045425d814c265ac16bcb19, State: Running, Role: LEADER
15:19:31.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.016445 29751 consensus_queue.cc:237] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:31.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.027616 29767 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7cf4979e8045425d814c265ac16bcb19. Latest consensus state: current_term: 2 leader_uuid: "7cf4979e8045425d814c265ac16bcb19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:31.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.029062 29767 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: This master's current role is: LEADER
15:19:31.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.038079 29772 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:19:31.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.049274 29772 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:19:31.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.055198 29772 catalog_manager.cc:1269] Loaded cluster ID: f403ce6f995c46b58cc12ed370c76892
15:19:31.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.055831 29772 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:19:31.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.061405 29772 catalog_manager.cc:1514] Loading token signing keys...
15:19:31.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.066540 29772 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19: Loaded TSK: 0
15:19:31.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.068463 29772 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:19:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.148600 29779 catalog_manager.cc:1269] Loaded cluster ID: f403ce6f995c46b58cc12ed370c76892
15:19:31.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.148905 29779 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: loading cluster ID for follower catalog manager: success
15:19:31.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.156738 29779 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: acquiring CA information for follower catalog manager: success
15:19:31.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:31 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045570, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.62@KRBTEST.COM for kudu/127.28.18.62@KRBTEST.COM
15:19:31.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.163246 29779 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:19:31.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:31 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045571, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:31.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.212034 29268 init.cc:295] Successfully reacquired a new kerberos TGT
15:19:31.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.231405 29638 master_runner.cc:186] Error getting master registration for 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111),
15:19:31.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.260838 29638 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:19:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.317788 29756 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:19:31.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.318506 29756 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:19:31.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.319597 29756 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:19:31.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.337173 29537 heartbeater.cc:344] Connected to a master server at 127.28.18.62:43993
15:19:31.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.344198 29716 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "3e065f11b1814304b8944abe3a89961d" instance_seqno: 1770045564023084) as {username='kudu', principal='kudu/127.28.18.3@KRBTEST.COM'} at 127.28.18.3:46993; Asking this server to re-register.
15:19:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.345968 29537 heartbeater.cc:461] Registering TS with master...
15:19:31.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.349423 29716 ts_manager.cc:194] Registered new tserver with Master: 3e065f11b1814304b8944abe3a89961d (127.28.18.3:34349)
15:19:31.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.369280 29756 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:19:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.369839 29756 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:19:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.370293 29756 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:19:31.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.370759 29756 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:19:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.371110 29756 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:19:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.371450 29756 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:19:31.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.371719 29756 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:40711/jwks.json
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.415551 29756 master_runner.cc:386] Master server non-default flags:
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.18.20:45329
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.18.62:43993,127.28.18.61:46833,127.28.18.60:43051
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/krb5kdc/kudu.keytab
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.28.18.60
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.18.60:43051
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:40711/jwks.json
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:19:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data/info.pb
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.18.60
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44537
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/logs
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:19:31.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 14:43:20 UTC on 5fd53c4cbb9d
15:19:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10313
15:19:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:19:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.417443 29588 consensus_peers.cc:597] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 -> Peer a24b594a8de94084a506d2643aeac833 (127.28.18.60:43051): Couldn't send request to peer a24b594a8de94084a506d2643aeac833. Status: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:19:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.418181 29756 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:19:31.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.419622 29756 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:19:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.433563 29791 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:31.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.635828 29726 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [term 2 FOLLOWER]: Refusing update from remote peer 7cf4979e8045425d814c265ac16bcb19: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
15:19:31.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.638113 29767 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [LEADER]: Connected to new peer: Peer: permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
15:19:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.673256 29797 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7cf4979e8045425d814c265ac16bcb19. Latest consensus state: current_term: 2 leader_uuid: "7cf4979e8045425d814c265ac16bcb19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:31.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.674117 29797 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: This master's current role is: FOLLOWER
15:19:31.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.686563 29767 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "7cf4979e8045425d814c265ac16bcb19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:31.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.687537 29767 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: This master's current role is: LEADER
15:19:31.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.698638 29797 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "7cf4979e8045425d814c265ac16bcb19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:31.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.699978 29797 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5f40696aca8e42b1bd0548ce27fbc279 [sys.catalog]: This master's current role is: FOLLOWER
15:19:31.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.886627 29716 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ea7cb7963e304e4b8938a5f263dd8621" instance_seqno: 1770045558300890) as {username='kudu', principal='kudu/127.28.18.1@KRBTEST.COM'} at 127.28.18.1:48953; Asking this server to re-register.
15:19:31.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.889249 29239 heartbeater.cc:461] Registering TS with master...
15:19:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.894254 29716 ts_manager.cc:194] Registered new tserver with Master: ea7cb7963e304e4b8938a5f263dd8621 (127.28.18.1:44473)
15:19:31.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.901175 29716 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "b65e8d3b11c04b48885d80fad2fc9f6a" instance_seqno: 1770045561446447) as {username='kudu', principal='kudu/127.28.18.2@KRBTEST.COM'} at 127.28.18.2:46875; Asking this server to re-register.
15:19:31.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.904052 29387 heartbeater.cc:461] Registering TS with master...
15:19:31.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:31.909701 29715 ts_manager.cc:194] Registered new tserver with Master: b65e8d3b11c04b48885d80fad2fc9f6a (127.28.18.2:45353)
15:19:32.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:32.345615 29536 heartbeater.cc:499] Master 127.28.18.61:46833 was elected leader, sending a full tablet report...
15:19:32.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:32.474859 29549 master_runner.cc:186] Error getting master registration for 127.28.18.60:43051: Network error: Client connection negotiation failed: client connection to 127.28.18.60:43051: connect: Connection refused (error 111),
15:19:32.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:32.475294 29549 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:19:32.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:32.837621 29236 heartbeater.cc:499] Master 127.28.18.61:46833 was elected leader, sending a full tablet report...
15:19:32.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:32.840561 29386 heartbeater.cc:499] Master 127.28.18.61:46833 was elected leader, sending a full tablet report...
15:19:33.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:32.835269 29790 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 29756
15:19:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:33.452158 29790 kernel_stack_watchdog.cc:198] Thread 29756 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
15:19:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:19:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:19:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:19:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:19:33.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:31.434823 29792 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:33.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:33.454277 29794 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:19:33.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:33.455042 29756 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.020s user 0.507s sys 0.862s
15:19:33.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:33.455574 29756 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.022s user 0.507s sys 0.862s
15:19:33.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:33.458052 29793 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2017 milliseconds
15:19:33.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.458065 29756 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:19:33.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.460386 29756 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:19:33.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.463929 29756 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:19:33.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.465396 29756 hybrid_clock.cc:648] HybridClock initialized: now 1770045573465279 us; error 131 us; skew 500 ppm
15:19:33.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.466301 29756 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:19:33.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:33 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045573, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:33.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.473610 29756 init.cc:377] Logged in from keytab as kudu/127.28.18.60@KRBTEST.COM (short username kudu)
15:19:33.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.480126 29756 webserver.cc:492] Webserver started at http://127.28.18.60:44537/ using document root <none> and password file <none>
15:19:33.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.481545 29756 fs_manager.cc:362] Metadata directory not provided
15:19:33.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.481868 29756 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:19:33.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.492764 29756 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.002s sys 0.006s
15:19:33.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.497838 29821 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:19:33.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.499091 29756 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
15:19:33.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.499475 29756 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data,/tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal
15:19:33.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a24b594a8de94084a506d2643aeac833"
15:19:33.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 15:19:12 on dist-test-slave-q1dj"
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.501713 29756 fs_report.cc:389] FS layout report
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/wal
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data/data
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:19:33.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:19:33.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.576395 29756 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.587880 29756 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:19:33.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.588467 29756 kserver.cc:163] Server-wide thread pool size limit: 3276
15:19:33.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.686949 29756 rpc_server.cc:307] RPC server started. Bound to: 127.28.18.60:43051
15:19:33.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.687000 29872 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.18.60:43051 every 8 connection(s)
15:19:33.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.692288 29756 server_base.cc:1192] Dumped server information to /tmp/dist-test-taskLsXfcC/test-tmp/mini-kudu-cluster6264630654382068801/master-2/data/info.pb
15:19:33.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.695573 28744 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu as pid 29756
15:19:33.696 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:19:33.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.714150 29873 sys_catalog.cc:263] Verifying existing consensus state
15:19:33.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.726393 29873 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: Bootstrap starting.
15:19:33.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.788482 29873 log.cc:826] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: Log is configured to *not* fsync() on all Append() calls
15:19:33.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:33 dist-test-slave-q1dj krb5kdc[28756](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045573, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:19:33.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.793118 29418 init.cc:295] Successfully reacquired a new kerberos TGT
15:19:33.802 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.18.61:46833(127.28.18.61:46833)] connecting to peer
15:19:33.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.815944 29873 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:19:33.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.817147 29873 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833: Bootstrap complete.
15:19:33.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.857983 29873 raft_consensus.cc:359] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:33.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.862226 29873 raft_consensus.cc:740] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: a24b594a8de94084a506d2643aeac833, State: Initialized, Role: FOLLOWER
15:19:33.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.863597 29873 consensus_queue.cc:260] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } }
15:19:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.932610 29878 sys_catalog.cc:455] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:33.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.933440 29878 sys_catalog.cc:458] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: This master's current role is: FOLLOWER
15:19:33.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:33 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045571, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.2@KRBTEST.COM for kudu/127.28.18.60@KRBTEST.COM
15:19:33.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.949752 29873 sys_catalog.cc:565] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: configured and running, proceeding with master startup.
15:19:33.950 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.18.62:43993(127.28.18.62:43993)] connecting to peer
15:19:33.954 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.28.18.60:43051(127.28.18.60:43051)] connecting to peer
15:19:33.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:33 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045566, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.61@KRBTEST.COM for kudu/127.28.18.60@KRBTEST.COM
15:19:33.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:33 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045567, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.1@KRBTEST.COM for kudu/127.28.18.60@KRBTEST.COM
15:19:33.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:33.997987 29385 heartbeater.cc:344] Connected to a master server at 127.28.18.60:43051
15:19:34.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.006299 29848 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [term 1 FOLLOWER]: Advancing to term 2
15:19:34.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.032573 29235 heartbeater.cc:344] Connected to a master server at 127.28.18.60:43051
15:19:34.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.041955 29893 sys_catalog.cc:455] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "7cf4979e8045425d814c265ac16bcb19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:34.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.043156 29893 sys_catalog.cc:458] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: This master's current role is: FOLLOWER
15:19:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.045919 29878 sys_catalog.cc:455] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 7cf4979e8045425d814c265ac16bcb19. Latest consensus state: current_term: 2 leader_uuid: "7cf4979e8045425d814c265ac16bcb19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.046548 29878 sys_catalog.cc:458] T 00000000000000000000000000000000 P a24b594a8de94084a506d2643aeac833 [sys.catalog]: This master's current role is: FOLLOWER
15:19:34.055 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.28.18.62:43993(127.28.18.62:43993)] Successfully connected to peer
15:19:34.059 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.18.61:46833(127.28.18.61:46833)] Successfully connected to peer
15:19:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.128844 29902 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:19:34.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:34 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045573, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.60@KRBTEST.COM for kudu/127.28.18.62@KRBTEST.COM
15:19:34.189 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.28.18.60:43051(127.28.18.60:43051)] Successfully connected to peer
15:19:34.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:34 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045573, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.60@KRBTEST.COM for kudu/127.28.18.61@KRBTEST.COM
15:19:34.236 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
15:19:34.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 15:19:34 dist-test-slave-q1dj krb5kdc[28756](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770045573, etypes {rep=17 tkt=17 ses=17}, kudu/127.28.18.60@KRBTEST.COM for kudu/127.28.18.60@KRBTEST.COM
15:19:34.248 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.18.61:46833(127.28.18.61:46833)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:19:34.248 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.28.18.62:43993(127.28.18.62:43993)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:19:34.251 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.18.61:46833: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:19:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:34.253823 29897 negotiation.cc:337] Failed RPC negotiation. Trace:
15:19:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.053653 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:45998 (local address 127.28.18.61:46833)
15:19:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.082935 (+ 29282us) server_negotiation.cc:207] Beginning negotiation
15:19:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.082963 (+ 28us) server_negotiation.cc:400] Waiting for connection header
15:19:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.145250 (+ 62287us) server_negotiation.cc:408] Connection header received
15:19:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.188958 (+ 43708us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
15:19:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.188989 (+ 31us) server_negotiation.cc:462] Received NEGOTIATE request from client
15:19:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.189659 (+ 670us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
15:19:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.189849 (+ 190us) server_negotiation.cc:227] Negotiated authn=SASL
15:19:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.252395 (+ 62546us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:45998: BlockingRecv error: recv got EOF from 127.0.0.1:45998 (error 108)
15:19:34.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":28686,"thread_start_us":28867,"threads_started":1}
15:19:34.255 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.28.18.60:43051(127.28.18.60:43051)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:19:34.256 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.28.18.60:43051: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:19:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:34.258985 29882 negotiation.cc:337] Failed RPC negotiation. Trace:
15:19:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.058916 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:51786 (local address 127.28.18.60:43051)
15:19:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.061399 (+ 2483us) server_negotiation.cc:207] Beginning negotiation
15:19:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.061426 (+ 27us) server_negotiation.cc:400] Waiting for connection header
15:19:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.189723 (+128297us) server_negotiation.cc:408] Connection header received
15:19:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.190491 (+ 768us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
15:19:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.190521 (+ 30us) server_negotiation.cc:462] Received NEGOTIATE request from client
15:19:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.190753 (+ 232us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
15:19:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.190893 (+ 140us) server_negotiation.cc:227] Negotiated authn=SASL
15:19:34.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.256824 (+ 65931us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:51786: BlockingRecv error: recv got EOF from 127.0.0.1:51786 (error 108)
15:19:34.260 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.28.18.62:43993: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:19:34.260 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.18.61:46833(127.28.18.61:46833)] handling channelInactive
15:19:34.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":1842}
15:19:34.261 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.28.18.60:43051(127.28.18.60:43051)] handling channelInactive
15:19:34.268 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=566), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.18.61:46833, [225ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.18.62:43993, [229ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.18.60:43051, [524ms] Sub RPC ConnectToMaster: received response from server master-127.28.18.62:43993: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [524ms] Sub RPC ConnectToMaster: received response from server master-127.28.18.61:46833: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [530ms] Sub RPC ConnectToMaster: received response from server master-127.28.18.60:43051: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.18.61:46833,127.28.18.62:43993,127.28.18.60:43051). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.18.61:46833,127.28.18.62:43993,127.28.18.60:43051). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
15:19:34.272 [DEBUG - kudu-nio-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=570), Traces: [0ms] refreshing cache from master, [54ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.18.61:46833, [225ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.18.62:43993, [229ms] Sub RPC ConnectToMaster: sending RPC to server master-127.28.18.60:43051, [524ms] Sub RPC ConnectToMaster: received response from server master-127.28.18.62:43993: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [524ms] Sub RPC ConnectToMaster: received response from server master-127.28.18.61:46833: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [530ms] Sub RPC ConnectToMaster: received response from server master-127.28.18.60:43051: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
15:19:34.276 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.18.61:46833(127.28.18.61:46833)] disconnecting while in state TERMINATED
15:19:34.279 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.18.62:43993(127.28.18.62:43993)] disconnecting while in state TERMINATED
15:19:34.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:34.279078 29896 negotiation.cc:337] Failed RPC negotiation. Trace:
15:19:34.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.052567 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:55808 (local address 127.28.18.62:43993)
15:19:34.279 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.28.18.60:43051(127.28.18.60:43051)] disconnecting while in state TERMINATED
15:19:34.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.081142 (+ 28575us) server_negotiation.cc:207] Beginning negotiation
15:19:34.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.081168 (+ 26us) server_negotiation.cc:400] Waiting for connection header
15:19:34.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.144891 (+ 63723us) server_negotiation.cc:408] Connection header received
15:19:34.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.188965 (+ 44074us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
15:19:34.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.188996 (+ 31us) server_negotiation.cc:462] Received NEGOTIATE request from client
15:19:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.189764 (+ 768us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
15:19:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.189929 (+ 165us) server_negotiation.cc:227] Negotiated authn=SASL
15:19:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 15:19:34.277432 (+ 87503us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:55808: BlockingRecv error: recv got EOF from 127.0.0.1:55808 (error 108)
15:19:34.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":27968,"thread_start_us":27714,"threads_started":1}
15:19:34.281 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.28.18.62:43993(127.28.18.62:43993)] handling channelInactive
15:19:34.282 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:19:34.284 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:19:34.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.289695 28744 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 29096
15:19:34.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.320106 28744 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 29250
15:19:34.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.357402 29799 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "7cf4979e8045425d814c265ac16bcb19" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "5f40696aca8e42b1bd0548ce27fbc279" member_type: VOTER last_known_addr { host: "127.28.18.62" port: 43993 } } peers { permanent_uuid: "7cf4979e8045425d814c265ac16bcb19" member_type: VOTER last_known_addr { host: "127.28.18.61" port: 46833 } } peers { permanent_uuid: "a24b594a8de94084a506d2643aeac833" member_type: VOTER last_known_addr { host: "127.28.18.60" port: 43051 } } }
15:19:34.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.359356 29799 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7cf4979e8045425d814c265ac16bcb19 [sys.catalog]: This master's current role is: LEADER
15:19:34.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.366853 28744 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 29399
15:19:34.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 15:19:34.402359 29588 connection.cc:537] server connection from 127.28.18.3:57203 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:19:34.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.403323 28744 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 29638
15:19:34.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.429951 28744 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 29549
15:19:34.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:34.465986 28744 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 29756
15:19:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T15:19:35Z chronyd exiting
15:19:36.690 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:19:36.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:36.691913 28009 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 28291
15:19:36.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:36.718441 28009 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 28455
15:19:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:36.744645 28009 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 28601
15:19:36.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:36.772083 28009 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 28021
15:19:36.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:36.804149 28009 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 28091
15:19:36.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 15:19:36.836467 28009 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskLsXfcC/build/tsan/bin/kudu with pid 28168
15:19:37.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T15:19:37Z chronyd exiting
15:19:39.019 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.18.61:46833,127.28.18.62:43993,127.28.18.60:43051). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.28.18.61:46833,127.28.18.62:43993,127.28.18.60:43051). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more