03:07:22.205 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:07:22.218 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasktpBrrC/test-tmp
03:07:22.222 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:07:22.226 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasktpBrrC/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:07:22.532 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:07:22.651 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:07:22.652 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:07:22.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:22.656059 27181 mini_oidc.cc:150] Starting JWKS server
03:07:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:22.659312 27181 webserver.cc:492] Webserver started at http://127.0.0.1:38029/ using document root <none> and password file <none>
03:07:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:22.659744 27181 mini_oidc.cc:186] Starting OIDC Discovery server
03:07:22.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:22.661429 27181 webserver.cc:492] Webserver started at http://127.0.0.1:41541/ using document root <none> and password file <none>
03:07:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T03:07:22Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:07:22.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T03:07:22Z Disabled control of system clock
03:07:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:22.724516 27181 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/wal
03:07:22.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/data
03:07:22.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:22.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:22.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:22.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/logs
03:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/data/info.pb
03:07:22.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:22.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:22.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:22.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:22.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:22.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:22.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:22.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.126:33837
03:07:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.126
03:07:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:22.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:22.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:22.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:22.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38029/jwks.json with env {}
03:07:23.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:23.036218 27193 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:23.036818 27193 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:23.037555 27193 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:23.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:23.070349 27193 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:23.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:23.070644 27193 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:23.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:23.070844 27193 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:23.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:23.071159 27193 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:23.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:23.071449 27193 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:23.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:23.071691 27193 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38029/jwks.json
03:07:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:23.108640 27193 master_runner.cc:386] Master server non-default flags:
03:07:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/data
03:07:23.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/wal
03:07:23.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:23.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:23.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:23.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.126:33837
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38029/jwks.json
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/data/info.pb
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.126
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:23.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/logs
03:07:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:23.110107 27193 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:23.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:23.112399 27193 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:23.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:23.125425 27199 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:24.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:23.125459 27200 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:24.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.358415 27201 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1230 milliseconds
03:07:24.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.360253 27202 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.362957 27193 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.238s user 0.388s sys 0.846s
03:07:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.363282 27193 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.238s user 0.391s sys 0.846s
03:07:24.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.363560 27193 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:24.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.365021 27193 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:24.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.371694 27193 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:24.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.373665 27193 hybrid_clock.cc:648] HybridClock initialized: now 1770001644373588 us; error 96 us; skew 500 ppm
03:07:24.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.374670 27193 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:24.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.383309 27193 webserver.cc:492] Webserver started at http://127.26.139.126:37647/ using document root <none> and password file <none>
03:07:24.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.384481 27193 fs_manager.cc:362] Metadata directory not provided
03:07:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.384768 27193 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:24.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.385365 27193 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:24.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.390090 27193 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/data/instance:
03:07:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f583ddc2ef64cd0bd09a9da53f7f489"
03:07:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:24 on dist-test-slave-0wq0"
03:07:24.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.391568 27193 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/wal/instance:
03:07:24.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f583ddc2ef64cd0bd09a9da53f7f489"
03:07:24.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:24 on dist-test-slave-0wq0"
03:07:24.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.400887 27193 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.000s sys 0.008s
03:07:24.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.407440 27209 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.408973 27193 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
03:07:24.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.409410 27193 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/wal
03:07:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f583ddc2ef64cd0bd09a9da53f7f489"
03:07:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:24 on dist-test-slave-0wq0"
03:07:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.409857 27193 fs_report.cc:389] FS layout report
03:07:24.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/wal
03:07:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/wal
03:07:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/data/data
03:07:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:24.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:24.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:24.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.490665 27193 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:24.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.492409 27193 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:24.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.493086 27193 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:24.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.566756 27193 rpc_server.cc:307] RPC server started. Bound to: 127.26.139.126:33837
03:07:24.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.566850 27260 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.139.126:33837 every 8 connection(s)
03:07:24.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.569712 27193 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/data/info.pb
03:07:24.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.573459 27181 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 27193
03:07:24.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.574039 27181 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-0/wal/instance
03:07:24.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.577543 27261 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:24.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.579282 27181 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:24.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:24.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/wal
03:07:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/data
03:07:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/logs
03:07:24.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/data/info.pb
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:24.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:24.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.125:43903
03:07:24.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.125
03:07:24.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:24.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:24.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:24.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:24.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:24.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:24.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:24.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:24.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38029/jwks.json with env {}
03:07:24.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.595387 27261 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } has no permanent_uuid. Determining permanent_uuid...
03:07:24.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.626210 27261 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } has no permanent_uuid. Determining permanent_uuid...
03:07:24.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.628961 27212 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.139.125:43903: connect: Connection refused (error 111)
03:07:24.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.633615 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.125:43903: Network error: Client connection negotiation failed: client connection to 127.26.139.125:43903: connect: Connection refused (error 111)
03:07:24.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.683156 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } attempt: 1
03:07:24.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.687120 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.125:43903: Network error: Client connection negotiation failed: client connection to 127.26.139.125:43903: connect: Connection refused (error 111)
03:07:24.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.755633 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } attempt: 2
03:07:24.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.760097 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.125:43903: Network error: Client connection negotiation failed: client connection to 127.26.139.125:43903: connect: Connection refused (error 111)
03:07:24.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.851514 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } attempt: 3
03:07:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.856106 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.125:43903: Network error: Client connection negotiation failed: client connection to 127.26.139.125:43903: connect: Connection refused (error 111)
03:07:24.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.916100 27263 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:24.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.916788 27263 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:24.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.917268 27263 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:24.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.953768 27263 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:24.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.954064 27263 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:24.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.954257 27263 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:24.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.954456 27263 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:24.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.954655 27263 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:24.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:24.954860 27263 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38029/jwks.json
03:07:24.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.993950 27263 master_runner.cc:386] Master server non-default flags:
03:07:24.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:24.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:24.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/data
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/wal
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.125:43903
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38029/jwks.json
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/data/info.pb
03:07:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.125
03:07:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/logs
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.995352 27263 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:24.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.997612 27263 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:24.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:24.999500 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } attempt: 4
03:07:25.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:25.003657 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.125:43903: Network error: Client connection negotiation failed: client connection to 127.26.139.125:43903: connect: Connection refused (error 111)
03:07:25.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:25.011806 27272 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:25.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:25.303093 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } attempt: 5
03:07:25.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:25.311169 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.125:43903: Network error: Client connection negotiation failed: client connection to 127.26.139.125:43903: connect: Connection refused (error 111)
03:07:25.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:25.858934 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } attempt: 6
03:07:25.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:25.871820 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.125:43903: Network error: Client connection negotiation failed: client connection to 127.26.139.125:43903: connect: Connection refused (error 111)
03:07:26.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:25.013815 27273 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:26.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.267408 27275 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:26.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.271620 27274 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1251 milliseconds
03:07:26.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.271729 27263 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:26.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.273339 27263 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:26.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.277208 27263 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:26.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.279086 27263 hybrid_clock.cc:648] HybridClock initialized: now 1770001646278892 us; error 47 us; skew 500 ppm
03:07:26.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.280053 27263 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:26.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.287671 27263 webserver.cc:492] Webserver started at http://127.26.139.125:36793/ using document root <none> and password file <none>
03:07:26.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.288823 27263 fs_manager.cc:362] Metadata directory not provided
03:07:26.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.289101 27263 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:26.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.289700 27263 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:26.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.294404 27263 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/data/instance:
03:07:26.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b160830488d414fad2bf8e00b9e2a9a"
03:07:26.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:26 on dist-test-slave-0wq0"
03:07:26.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.295895 27263 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/wal/instance:
03:07:26.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b160830488d414fad2bf8e00b9e2a9a"
03:07:26.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:26 on dist-test-slave-0wq0"
03:07:26.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.303835 27263 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
03:07:26.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.310770 27284 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:26.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.312117 27263 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
03:07:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.312539 27263 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/wal
03:07:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2b160830488d414fad2bf8e00b9e2a9a"
03:07:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:26 on dist-test-slave-0wq0"
03:07:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.313002 27263 fs_report.cc:389] FS layout report
03:07:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/wal
03:07:26.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/wal
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/data/data
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:26.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:26.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.368201 27263 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:26.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.370188 27263 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:26.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.370813 27263 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:26.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.475406 27263 rpc_server.cc:307] RPC server started. Bound to: 127.26.139.125:43903
03:07:26.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.475461 27335 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.139.125:43903 every 8 connection(s)
03:07:26.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.478564 27263 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/data/info.pb
03:07:26.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.486091 27336 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.487067 27181 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 27263
03:07:26.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.487553 27181 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-1/wal/instance
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.490613 27181 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/wal
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/data
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/logs
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/data/info.pb
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.124:40435
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.124
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:26.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:26.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:26.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:26.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38029/jwks.json with env {}
03:07:26.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.501762 27336 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } has no permanent_uuid. Determining permanent_uuid...
03:07:26.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.526471 27336 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } has no permanent_uuid. Determining permanent_uuid...
03:07:26.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.543012 27336 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } has no permanent_uuid. Determining permanent_uuid...
03:07:26.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.545475 27286 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:26.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.550462 27336 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:26.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.600220 27336 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 1
03:07:26.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.605562 27336 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:26.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.674021 27336 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 2
03:07:26.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.679165 27336 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:26.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.770633 27336 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 3
03:07:26.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.775588 27336 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:26.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.853677 27338 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:26.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.854377 27338 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:26.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.855134 27338 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:26.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.887039 27338 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:26.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.887578 27338 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:26.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.888091 27338 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:26.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.888506 27338 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.888859 27338 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:26.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.889110 27338 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38029/jwks.json
03:07:26.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.919201 27336 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 4
03:07:26.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.923591 27336 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:26.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.929701 27338 master_runner.cc:386] Master server non-default flags:
03:07:26.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:26.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:26.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:26.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:26.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/data
03:07:26.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/wal
03:07:26.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:26.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.124:40435
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38029/jwks.json
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/data/info.pb
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.124
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/logs
03:07:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:26.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:26.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:26.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:26.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:26.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:26.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:26.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.931499 27338 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:26.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.932505 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } attempt: 7
03:07:26.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.933735 27338 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:26.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.949932 27349 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:26.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:26.951979 27261 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } has no permanent_uuid. Determining permanent_uuid...
03:07:26.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.959034 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:27.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:27.017501 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 1
03:07:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:27.025580 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:27.107419 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 2
03:07:27.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:27.115147 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:27.200867 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 3
03:07:27.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:27.209159 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:27.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:27.223042 27336 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 5
03:07:27.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:27.230042 27336 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:27.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:27.349849 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 4
03:07:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:27.357751 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:27.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:27.641455 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 5
03:07:27.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:27.648798 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:27.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:27.777693 27336 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 6
03:07:27.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:27.791944 27336 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:28.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.201493 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 6
03:07:28.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:28.213833 27261 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.139.124:40435: Network error: Client connection negotiation failed: client connection to 127.26.139.124:40435: connect: Connection refused (error 111)
03:07:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:26.952283 27350 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:28.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:28.266664 27338 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.318s user 0.509s sys 0.805s
03:07:28.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:28.267207 27338 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.319s user 0.511s sys 0.805s
03:07:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.275743 27338 server_base.cc:1060] running on GCE node
03:07:28.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:28.277614 27359 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:28.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.279500 27338 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:28.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.283478 27338 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:28.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.285295 27338 hybrid_clock.cc:648] HybridClock initialized: now 1770001648285215 us; error 95 us; skew 500 ppm
03:07:28.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.286516 27338 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:28.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.296237 27338 webserver.cc:492] Webserver started at http://127.26.139.124:34055/ using document root <none> and password file <none>
03:07:28.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.297211 27338 fs_manager.cc:362] Metadata directory not provided
03:07:28.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.297423 27338 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:28.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.297854 27338 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:28.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.303256 27338 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/data/instance:
03:07:28.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77eb7add5e6c40c69624234a93d81a77"
03:07:28.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:28 on dist-test-slave-0wq0"
03:07:28.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.304414 27338 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/wal/instance:
03:07:28.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77eb7add5e6c40c69624234a93d81a77"
03:07:28.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:28 on dist-test-slave-0wq0"
03:07:28.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.313386 27338 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
03:07:28.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.321185 27364 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.322741 27338 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
03:07:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.323135 27338 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/wal
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "77eb7add5e6c40c69624234a93d81a77"
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:28 on dist-test-slave-0wq0"
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.323467 27338 fs_report.cc:389] FS layout report
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/wal
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/wal
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/data/data
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:28.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:28.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:28.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.422731 27338 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:28.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.424530 27338 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:28.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.425068 27338 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:28.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.522683 27338 rpc_server.cc:307] RPC server started. Bound to: 127.26.139.124:40435
03:07:28.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.522756 27415 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.139.124:40435 every 8 connection(s)
03:07:28.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.525574 27338 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/data/info.pb
03:07:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.527493 27181 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 27338
03:07:28.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.528069 27181 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/master-2/wal/instance
03:07:28.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.533949 27416 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:28.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.560038 27416 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } has no permanent_uuid. Determining permanent_uuid...
03:07:28.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.590143 27416 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } has no permanent_uuid. Determining permanent_uuid...
03:07:28.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.602840 27416 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } has no permanent_uuid. Determining permanent_uuid...
03:07:28.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.623570 27416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77: Bootstrap starting.
03:07:28.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.632050 27416 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77: Neither blocks nor log segments found. Creating new log.
03:07:28.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.633880 27416 log.cc:826] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77: Log is configured to *not* fsync() on all Append() calls
03:07:28.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.639784 27416 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77: No bootstrap required, opened a new log
03:07:28.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.662746 27416 raft_consensus.cc:359] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } }
03:07:28.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.663614 27416 raft_consensus.cc:385] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:28.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.663853 27416 raft_consensus.cc:740] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 77eb7add5e6c40c69624234a93d81a77, State: Initialized, Role: FOLLOWER
03:07:28.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.664793 27416 consensus_queue.cc:260] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } }
03:07:28.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.669198 27424 sys_catalog.cc:455] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } } }
03:07:28.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.670075 27424 sys_catalog.cc:458] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [sys.catalog]: This master's current role is: FOLLOWER
03:07:28.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.671164 27416 sys_catalog.cc:565] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [sys.catalog]: configured and running, proceeding with master startup.
03:07:28.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:28.681162 27370 tablet.cc:2378] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:28.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:28.706354 27435 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:28.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:28.706750 27435 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:28.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.696352 27338 master_runner.cc:186] Error getting master registration for 127.26.139.126:33837: OK, instance_id { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" instance_seqno: 1770001644546340 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:28.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.720386 27338 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:28.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.720813 27424 raft_consensus.cc:493] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:28.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.721402 27424 raft_consensus.cc:515] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } }
03:07:28.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.724185 27424 leader_election.cc:290] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9f583ddc2ef64cd0bd09a9da53f7f489 (127.26.139.126:33837), 2b160830488d414fad2bf8e00b9e2a9a (127.26.139.125:43903)
03:07:28.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.852511 27336 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 7
03:07:28.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.881055 27336 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a: Bootstrap starting.
03:07:28.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.888078 27336 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a: Neither blocks nor log segments found. Creating new log.
03:07:28.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.893296 27336 log.cc:826] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a: Log is configured to *not* fsync() on all Append() calls
03:07:28.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.900558 27336 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a: No bootstrap required, opened a new log
03:07:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.931505 27336 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } }
03:07:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.933163 27336 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:28.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.933490 27336 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2b160830488d414fad2bf8e00b9e2a9a, State: Initialized, Role: FOLLOWER
03:07:28.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.934451 27336 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } }
03:07:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.939409 27439 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } } }
03:07:28.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.940599 27439 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [sys.catalog]: This master's current role is: FOLLOWER
03:07:28.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.941370 27336 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [sys.catalog]: configured and running, proceeding with master startup.
03:07:28.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.948572 27311 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "77eb7add5e6c40c69624234a93d81a77" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2b160830488d414fad2bf8e00b9e2a9a" is_pre_election: true
03:07:28.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.949577 27311 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 77eb7add5e6c40c69624234a93d81a77 in term 0.
03:07:28.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.951067 27367 leader_election.cc:304] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2b160830488d414fad2bf8e00b9e2a9a, 77eb7add5e6c40c69624234a93d81a77; no voters:
03:07:28.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.952045 27424 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:07:28.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.952437 27424 raft_consensus.cc:493] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:07:28.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.952765 27424 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [term 0 FOLLOWER]: Advancing to term 1
03:07:28.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.960587 27424 raft_consensus.cc:515] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } }
03:07:28.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.963696 27424 leader_election.cc:290] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [CANDIDATE]: Term 1 election: Requested vote from peers 9f583ddc2ef64cd0bd09a9da53f7f489 (127.26.139.126:33837), 2b160830488d414fad2bf8e00b9e2a9a (127.26.139.125:43903)
03:07:28.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.965619 27311 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "77eb7add5e6c40c69624234a93d81a77" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2b160830488d414fad2bf8e00b9e2a9a"
03:07:28.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.966982 27311 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [term 0 FOLLOWER]: Advancing to term 1
03:07:28.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.985430 27311 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 77eb7add5e6c40c69624234a93d81a77 in term 1.
03:07:28.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.988231 27367 leader_election.cc:304] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2b160830488d414fad2bf8e00b9e2a9a, 77eb7add5e6c40c69624234a93d81a77; no voters:
03:07:28.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.989107 27424 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [term 1 FOLLOWER]: Leader election won for term 1
03:07:28.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.992028 27424 raft_consensus.cc:697] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [term 1 LEADER]: Becoming Leader. State: Replica: 77eb7add5e6c40c69624234a93d81a77, State: Running, Role: LEADER
03:07:28.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:28.993325 27424 consensus_queue.cc:237] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } }
03:07:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.006953 27445 sys_catalog.cc:455] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 77eb7add5e6c40c69624234a93d81a77. Latest consensus state: current_term: 1 leader_uuid: "77eb7add5e6c40c69624234a93d81a77" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } } }
03:07:29.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.008402 27445 sys_catalog.cc:458] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [sys.catalog]: This master's current role is: LEADER
03:07:29.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.016734 27448 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:07:29.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.024914 27448 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:07:29.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.046602 27311 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [term 1 FOLLOWER]: Refusing update from remote peer 77eb7add5e6c40c69624234a93d81a77: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:29.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.051357 27445 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:07:29.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.078141 27451 mvcc.cc:204] Tried to move back new op lower bound from 7249926754473189376 to 7249926754293710848. Current Snapshot: MvccSnapshot[applied={T|T < 7249926754473189376}]
03:07:29.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.080917 27439 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 77eb7add5e6c40c69624234a93d81a77. Latest consensus state: current_term: 1 leader_uuid: "77eb7add5e6c40c69624234a93d81a77" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } } }
03:07:29.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.085925 27439 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [sys.catalog]: This master's current role is: FOLLOWER
03:07:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.092358 27424 sys_catalog.cc:455] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "77eb7add5e6c40c69624234a93d81a77" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } } }
03:07:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.093307 27424 sys_catalog.cc:458] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [sys.catalog]: This master's current role is: LEADER
03:07:29.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.094857 27454 mvcc.cc:204] Tried to move back new op lower bound from 7249926754473189376 to 7249926754293710848. Current Snapshot: MvccSnapshot[applied={T|T < 7249926754473189376}]
03:07:29.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.098270 27448 catalog_manager.cc:1357] Generated new cluster ID: a843d11faf544bd1a479ce00742221a2
03:07:29.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.098696 27448 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:07:29.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.101400 27457 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:07:29.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.112895 27439 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "77eb7add5e6c40c69624234a93d81a77" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } } }
03:07:29.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.113643 27439 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a [sys.catalog]: This master's current role is: FOLLOWER
03:07:29.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.105477 27263 master_runner.cc:186] Error getting master registration for 127.26.139.126:33837: OK, instance_id { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" instance_seqno: 1770001644546340 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:29.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.139937 27263 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:29.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.147624 27448 catalog_manager.cc:1380] Generated new certificate authority record
03:07:29.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.150157 27448 catalog_manager.cc:1514] Loading token signing keys...
03:07:29.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.180823 27448 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77: Generated new TSK 0
03:07:29.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.181944 27448 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:07:29.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.195513 27181 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/wal
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/data
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/logs
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/data/info.pb
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:29.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.65:0
03:07:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.139.65
03:07:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.65
03:07:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:07:29.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.247294 27261 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } attempt: 7
03:07:29.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.271389 27261 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489: Bootstrap starting.
03:07:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.279551 27261 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489: Neither blocks nor log segments found. Creating new log.
03:07:29.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.282580 27261 log.cc:826] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489: Log is configured to *not* fsync() on all Append() calls
03:07:29.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.288864 27261 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489: No bootstrap required, opened a new log
03:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.321341 27261 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } }
03:07:29.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.322270 27261 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:29.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.322551 27261 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9f583ddc2ef64cd0bd09a9da53f7f489, State: Initialized, Role: FOLLOWER
03:07:29.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.323592 27261 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } }
03:07:29.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.330272 27261 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [sys.catalog]: configured and running, proceeding with master startup.
03:07:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.331089 27236 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "77eb7add5e6c40c69624234a93d81a77" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" is_pre_election: true
03:07:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.332051 27236 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 77eb7add5e6c40c69624234a93d81a77 in term 0.
03:07:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.330427 27235 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "77eb7add5e6c40c69624234a93d81a77" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489"
03:07:29.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.334965 27235 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [term 0 FOLLOWER]: Advancing to term 1
03:07:29.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.329428 27462 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } } }
03:07:29.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.348646 27462 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [sys.catalog]: This master's current role is: FOLLOWER
03:07:29.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.347956 27235 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 77eb7add5e6c40c69624234a93d81a77 in term 1.
03:07:29.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.351549 27234 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [term 1 FOLLOWER]: Refusing update from remote peer 77eb7add5e6c40c69624234a93d81a77: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:29.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.360227 27445 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:07:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.423081 27462 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 77eb7add5e6c40c69624234a93d81a77. Latest consensus state: current_term: 1 leader_uuid: "77eb7add5e6c40c69624234a93d81a77" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } } }
03:07:29.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.427732 27462 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [sys.catalog]: This master's current role is: FOLLOWER
03:07:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.425341 27467 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "77eb7add5e6c40c69624234a93d81a77" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } } }
03:07:29.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.428730 27467 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489 [sys.catalog]: This master's current role is: FOLLOWER
03:07:29.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.508844 27478 catalog_manager.cc:1269] Loaded cluster ID: a843d11faf544bd1a479ce00742221a2
03:07:29.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.519263 27478 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489: loading cluster ID for follower catalog manager: success
03:07:29.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.527128 27478 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489: acquiring CA information for follower catalog manager: success
03:07:29.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.538151 27478 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9f583ddc2ef64cd0bd09a9da53f7f489: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:29.785558 27459 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:29.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:29.786157 27459 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:29.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:29.787038 27459 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:29.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:29.821242 27459 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:29.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:29.822269 27459 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.139.65
03:07:29.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.861231 27459 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:29.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:29.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:29.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/data
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/wal
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.65:0
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/data/info.pb
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.65
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.139.65
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/logs
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:29.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:29.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:29.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:29.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:29.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:29.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:29.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:29.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:29.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:29.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.863059 27459 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:29.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.864986 27459 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:29.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:29.885898 27486 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.916002 27491 sys_catalog.cc:455] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "77eb7add5e6c40c69624234a93d81a77" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9f583ddc2ef64cd0bd09a9da53f7f489" member_type: VOTER last_known_addr { host: "127.26.139.126" port: 33837 } } peers { permanent_uuid: "2b160830488d414fad2bf8e00b9e2a9a" member_type: VOTER last_known_addr { host: "127.26.139.125" port: 43903 } } peers { permanent_uuid: "77eb7add5e6c40c69624234a93d81a77" member_type: VOTER last_known_addr { host: "127.26.139.124" port: 40435 } } }
03:07:29.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:29.926755 27491 sys_catalog.cc:458] T 00000000000000000000000000000000 P 77eb7add5e6c40c69624234a93d81a77 [sys.catalog]: This master's current role is: LEADER
03:07:30.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:30.133262 27457 catalog_manager.cc:1269] Loaded cluster ID: a843d11faf544bd1a479ce00742221a2
03:07:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:30.134878 27457 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a: loading cluster ID for follower catalog manager: success
03:07:30.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:30.170954 27457 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a: acquiring CA information for follower catalog manager: success
03:07:30.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:30.202632 27457 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 2b160830488d414fad2bf8e00b9e2a9a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:31.286974 27484 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 27459
03:07:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:29.892975 27485 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:31.812685 27484 kernel_stack_watchdog.cc:198] Thread 27459 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
03:07:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:31.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:31.813113 27459 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.925s user 0.501s sys 0.683s
03:07:31.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:31.813434 27459 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.925s user 0.501s sys 0.683s
03:07:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:31.820420 27487 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1931 milliseconds
03:07:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:31.821058 27488 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:31.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.821414 27459 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:31.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.823042 27459 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:31.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.835494 27459 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:31.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.837393 27459 hybrid_clock.cc:648] HybridClock initialized: now 1770001651837347 us; error 57 us; skew 500 ppm
03:07:31.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.838703 27459 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:31.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.868327 27459 webserver.cc:492] Webserver started at http://127.26.139.65:34839/ using document root <none> and password file <none>
03:07:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.870040 27459 fs_manager.cc:362] Metadata directory not provided
03:07:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.870587 27459 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:31.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.871397 27459 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.880895 27459 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/data/instance:
03:07:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54c7ed05e68d4482b0deaf950814af3e"
03:07:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:31 on dist-test-slave-0wq0"
03:07:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.893705 27459 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/wal/instance:
03:07:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54c7ed05e68d4482b0deaf950814af3e"
03:07:31.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:31 on dist-test-slave-0wq0"
03:07:31.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.916523 27459 fs_manager.cc:696] Time spent creating directory manager: real 0.021s user 0.004s sys 0.015s
03:07:31.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.930330 27500 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.931865 27459 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.006s sys 0.004s
03:07:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.932288 27459 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/wal
03:07:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54c7ed05e68d4482b0deaf950814af3e"
03:07:31.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:31 on dist-test-slave-0wq0"
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:31.932760 27459 fs_report.cc:389] FS layout report
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/wal
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/wal
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/data/data
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:31.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:32.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.036911 27459 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:32.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.039031 27459 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:32.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.039608 27459 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:32.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.049185 27459 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.061405 27459 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.061726 27459 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.062024 27459 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:32.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.062238 27459 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:32.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.916023 27613 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.139.65:45453 every 8 connection(s)
03:07:32.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.916626 27459 rpc_server.cc:307] RPC server started. Bound to: 127.26.139.65:45453
03:07:32.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.920270 27459 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/data/info.pb
03:07:32.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.925182 27181 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 27459
03:07:32.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.925836 27181 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-0/wal/instance
03:07:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:32.951018 27181 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/wal
03:07:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/data
03:07:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/logs
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/data/info.pb
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.66:0
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.139.66
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.66
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:07:33.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.015713 27614 heartbeater.cc:344] Connected to a master server at 127.26.139.124:40435
03:07:33.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.016278 27614 heartbeater.cc:461] Registering TS with master...
03:07:33.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.017822 27614 heartbeater.cc:507] Master 127.26.139.124:40435 requested a full tablet report, sending...
03:07:33.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.021229 27381 ts_manager.cc:194] Registered new tserver with Master: 54c7ed05e68d4482b0deaf950814af3e (127.26.139.65:45453)
03:07:33.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.024051 27381 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.139.65:40093
03:07:33.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.041263 27617 heartbeater.cc:344] Connected to a master server at 127.26.139.125:43903
03:07:33.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.041834 27617 heartbeater.cc:461] Registering TS with master...
03:07:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.042523 27617 heartbeater.cc:507] Master 127.26.139.125:43903 requested a full tablet report, sending...
03:07:33.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.044552 27619 heartbeater.cc:344] Connected to a master server at 127.26.139.126:33837
03:07:33.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.045796 27619 heartbeater.cc:461] Registering TS with master...
03:07:33.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.046525 27619 heartbeater.cc:507] Master 127.26.139.126:33837 requested a full tablet report, sending...
03:07:33.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.050756 27226 ts_manager.cc:194] Registered new tserver with Master: 54c7ed05e68d4482b0deaf950814af3e (127.26.139.65:45453)
03:07:33.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.051018 27301 ts_manager.cc:194] Registered new tserver with Master: 54c7ed05e68d4482b0deaf950814af3e (127.26.139.65:45453)
03:07:33.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:33.482003 27618 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:33.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:33.482672 27618 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:33.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:33.483824 27618 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:33.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:33.531057 27618 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:33.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:33.532089 27618 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.139.66
03:07:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.576613 27618 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:33.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/data
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/wal
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.66:0
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/data/info.pb
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.66
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.139.66
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/logs
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:33.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:33.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.578943 27618 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:33.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:33.581082 27618 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:33.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:33.599354 27631 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:34.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.028846 27614 heartbeater.cc:499] Master 127.26.139.124:40435 was elected leader, sending a full tablet report...
03:07:34.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:33.602851 27632 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:34.946825 27618 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.346s user 0.555s sys 0.777s
03:07:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:34.947157 27618 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.347s user 0.558s sys 0.777s
03:07:34.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.949105 27618 server_base.cc:1060] running on GCE node
03:07:34.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:34.951215 27637 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:34.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.952947 27618 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:34.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.955999 27618 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:34.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.957532 27618 hybrid_clock.cc:648] HybridClock initialized: now 1770001654957460 us; error 53 us; skew 500 ppm
03:07:34.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.958788 27618 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:34.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.968490 27618 webserver.cc:492] Webserver started at http://127.26.139.66:38601/ using document root <none> and password file <none>
03:07:34.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.970016 27618 fs_manager.cc:362] Metadata directory not provided
03:07:34.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.970366 27618 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:34.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.970988 27618 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.978503 27618 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/data/instance:
03:07:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0cc4d6c161b041a1af22576cecdca031"
03:07:34.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:34 on dist-test-slave-0wq0"
03:07:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.980280 27618 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/wal/instance:
03:07:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0cc4d6c161b041a1af22576cecdca031"
03:07:34.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:34 on dist-test-slave-0wq0"
03:07:34.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:34.992079 27618 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.010s sys 0.001s
03:07:35.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.000373 27642 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.001775 27618 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.001s
03:07:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.002197 27618 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/wal
03:07:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0cc4d6c161b041a1af22576cecdca031"
03:07:35.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:34 on dist-test-slave-0wq0"
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.002664 27618 fs_report.cc:389] FS layout report
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/wal
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/wal
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/data/data
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:35.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:35.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.101248 27618 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:35.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.102737 27618 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:35.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.103215 27618 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:35.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.105683 27618 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:35.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.109928 27618 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:35.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.110169 27618 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:35.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.110532 27618 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:35.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.110715 27618 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:35.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.274291 27618 rpc_server.cc:307] RPC server started. Bound to: 127.26.139.66:37693
03:07:35.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.274391 27754 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.139.66:37693 every 8 connection(s)
03:07:35.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.276945 27618 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/data/info.pb
03:07:35.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.280084 27181 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 27618
03:07:35.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.280609 27181 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-1/wal/instance
03:07:35.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.302348 27181 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/wal
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/data
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/logs
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/data/info.pb
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:35.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.67:0
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.139.67
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.67
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:35.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:07:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.348877 27757 heartbeater.cc:344] Connected to a master server at 127.26.139.126:33837
03:07:35.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.349601 27757 heartbeater.cc:461] Registering TS with master...
03:07:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.348884 27755 heartbeater.cc:344] Connected to a master server at 127.26.139.124:40435
03:07:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.350441 27755 heartbeater.cc:461] Registering TS with master...
03:07:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.350847 27756 heartbeater.cc:344] Connected to a master server at 127.26.139.125:43903
03:07:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.351094 27757 heartbeater.cc:507] Master 127.26.139.126:33837 requested a full tablet report, sending...
03:07:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.351250 27756 heartbeater.cc:461] Registering TS with master...
03:07:35.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.351632 27755 heartbeater.cc:507] Master 127.26.139.124:40435 requested a full tablet report, sending...
03:07:35.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.352176 27756 heartbeater.cc:507] Master 127.26.139.125:43903 requested a full tablet report, sending...
03:07:35.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.354627 27226 ts_manager.cc:194] Registered new tserver with Master: 0cc4d6c161b041a1af22576cecdca031 (127.26.139.66:37693)
03:07:35.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.355217 27381 ts_manager.cc:194] Registered new tserver with Master: 0cc4d6c161b041a1af22576cecdca031 (127.26.139.66:37693)
03:07:35.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.355746 27301 ts_manager.cc:194] Registered new tserver with Master: 0cc4d6c161b041a1af22576cecdca031 (127.26.139.66:37693)
03:07:35.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.357020 27381 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.139.66:42225
03:07:35.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:35.677352 27765 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:35.677943 27765 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:35.678550 27765 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:35.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:35.712067 27765 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:35.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:35.712924 27765 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.139.67
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.749138 27765 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.139.84:34067
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/data
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/wal
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.139.67:0
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:35.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/data/info.pb
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.139.67
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.139.126:33837,127.26.139.125:43903,127.26.139.124:40435
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.139.67
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/logs
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:35.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:35.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:35.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:35.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:35.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:35.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:35.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.751003 27765 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:35.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:35.752717 27765 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:35.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:35.766782 27771 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:36.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:36.360285 27755 heartbeater.cc:499] Master 127.26.139.124:40435 was elected leader, sending a full tablet report...
03:07:37.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:35.769711 27772 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:37.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:35.778487 27774 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:37.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:37.000401 27773 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
03:07:37.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.000923 27765 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:37.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.004698 27765 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:37.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.007018 27765 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:37.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.008422 27765 hybrid_clock.cc:648] HybridClock initialized: now 1770001657008388 us; error 40 us; skew 500 ppm
03:07:37.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.009310 27765 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:37.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.016404 27765 webserver.cc:492] Webserver started at http://127.26.139.67:34785/ using document root <none> and password file <none>
03:07:37.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.017809 27765 fs_manager.cc:362] Metadata directory not provided
03:07:37.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.018075 27765 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:37.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.018492 27765 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:37.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.022891 27765 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/data/instance:
03:07:37.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "940cfc0e65694e9cbd25acfd7f1a3d79"
03:07:37.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:37 on dist-test-slave-0wq0"
03:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.024312 27765 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/wal/instance:
03:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "940cfc0e65694e9cbd25acfd7f1a3d79"
03:07:37.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:37 on dist-test-slave-0wq0"
03:07:37.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.031622 27765 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
03:07:37.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.038211 27784 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:37.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.039703 27765 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
03:07:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.040048 27765 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/wal
03:07:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "940cfc0e65694e9cbd25acfd7f1a3d79"
03:07:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:37 on dist-test-slave-0wq0"
03:07:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.040355 27765 fs_report.cc:389] FS layout report
03:07:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/wal
03:07:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/wal
03:07:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/data/data
03:07:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:37.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:37.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:37.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:37.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:37.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:37.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:37.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:37.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:37.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:37.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.100795 27765 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:37.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.102267 27765 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:37.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.102705 27765 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:37.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.105281 27765 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:37.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.109766 27765 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:37.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.109995 27765 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:37.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.110195 27765 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:37.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.110353 27765 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:37.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.280834 27765 rpc_server.cc:307] RPC server started. Bound to: 127.26.139.67:39149
03:07:37.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.281018 27896 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.139.67:39149 every 8 connection(s)
03:07:37.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.285286 27765 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/data/info.pb
03:07:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.290089 27181 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 27765
03:07:37.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.290611 27181 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster1778866820402715578/ts-2/wal/instance
03:07:37.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.332214 27897 heartbeater.cc:344] Connected to a master server at 127.26.139.124:40435
03:07:37.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.331506 27898 heartbeater.cc:344] Connected to a master server at 127.26.139.125:43903
03:07:37.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.332837 27897 heartbeater.cc:461] Registering TS with master...
03:07:37.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.332899 27898 heartbeater.cc:461] Registering TS with master...
03:07:37.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.335042 27897 heartbeater.cc:507] Master 127.26.139.124:40435 requested a full tablet report, sending...
03:07:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.336287 27898 heartbeater.cc:507] Master 127.26.139.125:43903 requested a full tablet report, sending...
03:07:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.336791 27901 heartbeater.cc:344] Connected to a master server at 127.26.139.126:33837
03:07:37.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.337145 27901 heartbeater.cc:461] Registering TS with master...
03:07:37.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.338209 27901 heartbeater.cc:507] Master 127.26.139.126:33837 requested a full tablet report, sending...
03:07:37.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.338526 27381 ts_manager.cc:194] Registered new tserver with Master: 940cfc0e65694e9cbd25acfd7f1a3d79 (127.26.139.67:39149)
03:07:37.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.340469 27301 ts_manager.cc:194] Registered new tserver with Master: 940cfc0e65694e9cbd25acfd7f1a3d79 (127.26.139.67:39149)
03:07:37.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.341282 27381 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.26.139.67:58477
03:07:37.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.341621 27226 ts_manager.cc:194] Registered new tserver with Master: 940cfc0e65694e9cbd25acfd7f1a3d79 (127.26.139.67:39149)
03:07:37.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:37.345956 27181 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
03:07:37.358 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:07:37.360 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:07:37.374 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.26.139.126"
port: 33837
}
bound_http_address {
host: "127.26.139.126"
port: 37647
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.26.139.125"
port: 43903
}
bound_http_address {
host: "127.26.139.125"
port: 36793
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.26.139.124"
port: 40435
}
bound_http_address {
host: "127.26.139.124"
port: 34055
}
}
}
03:07:37.377 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:07:37.380 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.26.139.65"
port: 45453
}
bound_http_address {
host: "127.26.139.65"
port: 34839
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.26.139.66"
port: 37693
}
bound_http_address {
host: "127.26.139.66"
port: 38601
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.26.139.67"
port: 39149
}
bound_http_address {
host: "127.26.139.67"
port: 34785
}
}
}
03:07:37.380 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:07:37.561 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
03:07:37.591 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:07:37.720 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-tasktpBrrC/test-tmp
03:07:37.720 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:07:37.721 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-tasktpBrrC/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:07:37.724 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:07:38.089 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:38.089 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:07:38.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:38.093621 27908 mini_oidc.cc:150] Starting JWKS server
03:07:38.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:38.097959 27908 webserver.cc:492] Webserver started at http://127.0.0.1:42881/ using document root <none> and password file <none>
03:07:38.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:38.098502 27908 mini_oidc.cc:186] Starting OIDC Discovery server
03:07:38.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:38.101853 27908 webserver.cc:492] Webserver started at http://127.0.0.1:39475/ using document root <none> and password file <none>
03:07:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:07:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/principal' for realm 'KRBTEST.COM',
03:07:38.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:07:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:38 dist-test-slave-0wq0 krb5kdc[27921](info): setting up network...
03:07:38.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:07:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:38 dist-test-slave-0wq0 krb5kdc[27921](info): set up 2 sockets
03:07:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:38 dist-test-slave-0wq0 krb5kdc[27921](info): commencing operation
03:07:38.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:07:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:38.345547 27897 heartbeater.cc:499] Master 127.26.139.124:40435 was elected leader, sending a full tablet report...
03:07:40.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:40.297513 27908 mini_kdc.cc:121] Time spent starting KDC: real 2.156s user 0.009s sys 0.012s
03:07:40.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:07:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:40.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:07:40.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:07:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:40.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:07:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:07:40.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:40.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:07:40.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:07:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/test-user.keytab.
03:07:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/test-user.keytab.
03:07:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:40 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001660, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:40.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T03:07:40Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:07:40.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T03:07:40Z Disabled control of system clock
03:07:40.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.65.62@KRBTEST.COM; defaulting to no policy
03:07:40.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:40.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.65.62@KRBTEST.COM" created.
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:40.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.65.62@KRBTEST.COM; defaulting to no policy
03:07:40.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:40.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.65.62@KRBTEST.COM" created.
03:07:40.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:40.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:40.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:40.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:40.556463 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/logs
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/info.pb
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:40.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.62:41407
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.62
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.62
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:07:40.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:40.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:40.911104 27938 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:40.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:40.911798 27938 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:40.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:40.912576 27938 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:40.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:40.945338 27938 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:40.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:40.945657 27938 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:40.946041 27938 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:40.946375 27938 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:40.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:40.946707 27938 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:40.947054 27938 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:40.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:40.947301 27938 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42881/jwks.json
03:07:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:40.987149 27938 master_runner.cc:386] Master server non-default flags:
03:07:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data
03:07:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:07:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:40.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.62
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.62:41407
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:40.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/info.pb
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.62
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/logs
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:40.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:40.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:40.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:40.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:40.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:40.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:40.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:40.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:40.988807 27938 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:40.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:40.990211 27938 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:41.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:41.002235 27944 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:41.195009 27257 debug-util.cc:398] Leaking SignalData structure 0x7b080009e160 after lost signal to thread 27194
03:07:41.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:41.196022 27257 debug-util.cc:398] Leaking SignalData structure 0x7b08000a2b00 after lost signal to thread 27260
03:07:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:42.414039 27943 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 27938
03:07:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:42.698786 27943 kernel_stack_watchdog.cc:198] Thread 27938 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
03:07:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:42.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:41.004230 27945 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:42.700126 27938 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.697s user 0.498s sys 0.889s
03:07:42.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:42.700515 27946 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1696 milliseconds
03:07:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:42.700659 27938 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.697s user 0.499s sys 0.889s
03:07:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:42.701082 27947 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:42.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.701215 27938 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:42.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.704919 27938 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:42.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.708032 27938 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:42.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.709748 27938 hybrid_clock.cc:648] HybridClock initialized: now 1770001662709691 us; error 72 us; skew 500 ppm
03:07:42.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.710677 27938 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:42.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:42 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001662, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:42.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.717437 27938 init.cc:377] Logged in from keytab as kudu/127.27.65.62@KRBTEST.COM (short username kudu)
03:07:42.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.724730 27938 webserver.cc:492] Webserver started at http://127.27.65.62:36607/ using document root <none> and password file <none>
03:07:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.726163 27938 fs_manager.cc:362] Metadata directory not provided
03:07:42.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.726562 27938 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:42.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.727208 27938 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:42.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.734431 27938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/instance:
03:07:42.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "040a41d80f044969aaa60adb75a339c5"
03:07:42.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:42 on dist-test-slave-0wq0"
03:07:42.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.736189 27938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal/instance:
03:07:42.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "040a41d80f044969aaa60adb75a339c5"
03:07:42.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:42 on dist-test-slave-0wq0"
03:07:42.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.746872 27938 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.005s sys 0.005s
03:07:42.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.755767 27956 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:42.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.757294 27938 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
03:07:42.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.757794 27938 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:07:42.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "040a41d80f044969aaa60adb75a339c5"
03:07:42.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:42 on dist-test-slave-0wq0"
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.758605 27938 fs_report.cc:389] FS layout report
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/data
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:42.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:42.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.838763 27938 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.847788 27938 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:42.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.848390 27938 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.966883 27938 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.62:41407
03:07:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.966977 28007 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.62:41407 every 8 connection(s)
03:07:42.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.970062 27938 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/info.pb
03:07:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.972258 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 27938
03:07:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.972867 27908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal/instance
03:07:42.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:42.982262 28008 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:42.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.65.61@KRBTEST.COM; defaulting to no policy
03:07:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:43.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.65.61@KRBTEST.COM" created.
03:07:43.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:43.000177 28008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } has no permanent_uuid. Determining permanent_uuid...
03:07:43.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:43 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001662, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.62@KRBTEST.COM for kudu/127.27.65.62@KRBTEST.COM
03:07:43.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:43.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:43.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:43.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:43.047917 28008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } has no permanent_uuid. Determining permanent_uuid...
03:07:43.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.65.61@KRBTEST.COM; defaulting to no policy
03:07:43.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.050357 27959 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.65.61:42281: connect: Connection refused (error 111)
03:07:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:43.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.65.61@KRBTEST.COM" created.
03:07:43.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.056764 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.61:42281: Network error: Client connection negotiation failed: client connection to 127.27.65.61:42281: connect: Connection refused (error 111)
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:43.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:43.077462 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:07:43.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/logs
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/info.pb
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.61:42281
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.61
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.61
03:07:43.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:43.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:43.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:43.106370 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } attempt: 1
03:07:43.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.111099 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.61:42281: Network error: Client connection negotiation failed: client connection to 127.27.65.61:42281: connect: Connection refused (error 111)
03:07:43.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:43.179545 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } attempt: 2
03:07:43.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.184949 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.61:42281: Network error: Client connection negotiation failed: client connection to 127.27.65.61:42281: connect: Connection refused (error 111)
03:07:43.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:43.279498 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } attempt: 3
03:07:43.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.293890 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.61:42281: Network error: Client connection negotiation failed: client connection to 127.27.65.61:42281: connect: Connection refused (error 111)
03:07:43.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:43.437518 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } attempt: 4
03:07:43.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.443625 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.61:42281: Network error: Client connection negotiation failed: client connection to 127.27.65.61:42281: connect: Connection refused (error 111)
03:07:43.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.454464 28017 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:43.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.455202 28017 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:43.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.456108 28017 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:43.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.492717 28017 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:43.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.493049 28017 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:43.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.493429 28017 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:43.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.493772 28017 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:43.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.494105 28017 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:43.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.494544 28017 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:43.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.494824 28017 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42881/jwks.json
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:43.546669 28017 master_runner.cc:386] Master server non-default flags:
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.61
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.61:42281
03:07:43.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/info.pb
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.61
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/logs
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:43.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:43.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:43.550251 28017 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:43.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:43.551993 28017 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:43.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.566762 28025 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:43.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:43.743083 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } attempt: 5
03:07:43.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.748998 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.61:42281: Network error: Client connection negotiation failed: client connection to 127.27.65.61:42281: connect: Connection refused (error 111)
03:07:44.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:44.296496 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } attempt: 6
03:07:44.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:44.345335 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.61:42281: Network error: Client connection negotiation failed: client connection to 127.27.65.61:42281: connect: Connection refused (error 111)
03:07:45.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.406153 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } attempt: 7
03:07:45.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:45.424235 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.61:42281: Network error: Client connection negotiation failed: client connection to 127.27.65.61:42281: connect: Connection refused (error 111)
03:07:45.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:44.967375 28024 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 28017
03:07:45.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:45.487830 28024 kernel_stack_watchdog.cc:198] Thread 28017 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
03:07:45.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:45.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:45.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:45.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:45.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:45.488960 28017 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.922s user 0.585s sys 1.162s
03:07:45.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:45.489858 28017 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.923s user 0.585s sys 1.162s
03:07:45.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:43.567587 28026 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:45.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:45.491922 28028 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:45.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:45.497318 28027 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1915 milliseconds
03:07:45.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.497323 28017 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:45.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.499068 28017 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:45.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.502179 28017 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:45.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.503713 28017 hybrid_clock.cc:648] HybridClock initialized: now 1770001665503584 us; error 135 us; skew 500 ppm
03:07:45.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.504789 28017 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:45.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:45 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001665, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:45.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.512645 28017 init.cc:377] Logged in from keytab as kudu/127.27.65.61@KRBTEST.COM (short username kudu)
03:07:45.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.520248 28017 webserver.cc:492] Webserver started at http://127.27.65.61:36875/ using document root <none> and password file <none>
03:07:45.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.521591 28017 fs_manager.cc:362] Metadata directory not provided
03:07:45.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.521874 28017 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:45.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.522409 28017 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:45.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.527120 28017 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/instance:
03:07:45.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d360522e835469d8de898131ddaaafc"
03:07:45.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:45 on dist-test-slave-0wq0"
03:07:45.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.528359 28017 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal/instance:
03:07:45.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d360522e835469d8de898131ddaaafc"
03:07:45.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:45 on dist-test-slave-0wq0"
03:07:45.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.537127 28017 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
03:07:45.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.544090 28042 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.545405 28017 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.002s
03:07:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.545830 28017 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:07:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d360522e835469d8de898131ddaaafc"
03:07:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:45 on dist-test-slave-0wq0"
03:07:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.546226 28017 fs_report.cc:389] FS layout report
03:07:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:07:45.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/data
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:45.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:45.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.612644 28017 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:45.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.621507 28017 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:45.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.622071 28017 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.708825 28017 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.61:42281
03:07:45.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.708863 28094 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.61:42281 every 8 connection(s)
03:07:45.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.711699 28017 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/info.pb
03:07:45.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.715873 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 28017
03:07:45.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.716337 27908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal/instance
03:07:45.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.719696 28095 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:45.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.65.60@KRBTEST.COM; defaulting to no policy
03:07:45.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.731489 28095 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } has no permanent_uuid. Determining permanent_uuid...
03:07:45.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:45.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.65.60@KRBTEST.COM" created.
03:07:45.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:45 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001665, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.61@KRBTEST.COM for kudu/127.27.65.62@KRBTEST.COM
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:45.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:45.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.772117 28095 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } has no permanent_uuid. Determining permanent_uuid...
03:07:45.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.65.60@KRBTEST.COM; defaulting to no policy
03:07:45.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:45.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.65.60@KRBTEST.COM" created.
03:07:45.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:45 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001665, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.61@KRBTEST.COM for kudu/127.27.65.61@KRBTEST.COM
03:07:45.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.803831 28095 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } has no permanent_uuid. Determining permanent_uuid...
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:45.806278 28045 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:45.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:45.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:45.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.808790 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:45.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:45.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:07:45.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data
03:07:45.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:45.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:45.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:45.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/logs
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/info.pb
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.60:38795
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.60
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.60
03:07:45.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:45.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:45.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:45.811609 28095 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.60:38795: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:45.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.861193 28095 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } attempt: 1
03:07:45.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:45.865549 28095 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.60:38795: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:45.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:45.933970 28095 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } attempt: 2
03:07:45.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:45.938767 28095 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.60:38795: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:46.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:46.030310 28095 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } attempt: 3
03:07:46.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.034371 28095 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.60:38795: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:46.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.160640 28105 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:46.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.161193 28105 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:46.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.161866 28105 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:46.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:46.177878 28095 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } attempt: 4
03:07:46.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.182320 28095 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.60:38795: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:46.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.196238 28105 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:46.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.196542 28105 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:46.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.196823 28105 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:46.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.197026 28105 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:46.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.197217 28105 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:46.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.197408 28105 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:46.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.197549 28105 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42881/jwks.json
03:07:46.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:46.238169 28105 master_runner.cc:386] Master server non-default flags:
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.60
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.60:38795
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/info.pb
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.60
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:46.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/logs
03:07:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:46.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:46.239655 28105 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:46.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:46.240880 28105 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:46.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.256923 28111 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:46.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:46.481849 28095 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } attempt: 5
03:07:46.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.489717 28095 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.60:38795: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:47.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.037317 28095 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } attempt: 6
03:07:47.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:47.048686 28095 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.60:38795: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:47.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.514807 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } attempt: 8
03:07:47.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:47 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001662, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.62@KRBTEST.COM for kudu/127.27.65.61@KRBTEST.COM
03:07:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.591992 28008 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } has no permanent_uuid. Determining permanent_uuid...
03:07:47.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:47.600582 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.60:38795: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:47.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.260080 28112 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:47.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:46.266978 28114 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:47.631129 28113 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1367 milliseconds
03:07:47.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.631259 28105 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:47.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.632699 28105 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:47.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.635354 28105 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.636768 28105 hybrid_clock.cc:648] HybridClock initialized: now 1770001667636725 us; error 44 us; skew 500 ppm
03:07:47.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.637642 28105 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:47.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:47 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001667, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:47.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.644402 28105 init.cc:377] Logged in from keytab as kudu/127.27.65.60@KRBTEST.COM (short username kudu)
03:07:47.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.650384 28105 webserver.cc:492] Webserver started at http://127.27.65.60:41997/ using document root <none> and password file <none>
03:07:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.651505 28105 fs_manager.cc:362] Metadata directory not provided
03:07:47.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.651762 28105 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:47.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.652173 28105 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:47.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.656807 28105 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/instance:
03:07:47.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "080b9f8181db40ccb2df34a96fcb4738"
03:07:47.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:47 on dist-test-slave-0wq0"
03:07:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.657944 28105 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal/instance:
03:07:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "080b9f8181db40ccb2df34a96fcb4738"
03:07:47.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:47 on dist-test-slave-0wq0"
03:07:47.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.666273 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } attempt: 1
03:07:47.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.667071 28105 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.007s sys 0.004s
03:07:47.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:47.671722 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.60:38795: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:47.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.676872 28128 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:47.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.678231 28105 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
03:07:47.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.678580 28105 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:07:47.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "080b9f8181db40ccb2df34a96fcb4738"
03:07:47.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:47 on dist-test-slave-0wq0"
03:07:47.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.679049 28105 fs_report.cc:389] FS layout report
03:07:47.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:47.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:07:47.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/data
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:47.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:47.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.725178 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } attempt: 2
03:07:47.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:47.730427 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.60:38795: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:47.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.737769 28105 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:47.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.744208 28105 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:47.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.744750 28105 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:47.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.806978 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } attempt: 3
03:07:47.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:47.812542 28008 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.65.60:38795: Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:07:47.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.842587 28105 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.60:38795
03:07:47.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.842662 28179 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.60:38795 every 8 connection(s)
03:07:47.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.845499 28105 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/info.pb
03:07:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.849052 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 28105
03:07:47.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.849558 27908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal/instance
03:07:47.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.854662 28180 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:07:47.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.874432 28180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } has no permanent_uuid. Determining permanent_uuid...
03:07:47.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:47 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001660, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.65.62@KRBTEST.COM
03:07:47.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:47 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001667, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.60@KRBTEST.COM for kudu/127.27.65.62@KRBTEST.COM
03:07:47.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:47 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001660, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.65.61@KRBTEST.COM
03:07:47.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.924273 28180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } has no permanent_uuid. Determining permanent_uuid...
03:07:47.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:47 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001667, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.60@KRBTEST.COM for kudu/127.27.65.61@KRBTEST.COM
03:07:47.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:47 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001660, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:07:47.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.952297 28180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } has no permanent_uuid. Determining permanent_uuid...
03:07:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:47 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001667, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.60@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:07:47.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.967986 28008 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } attempt: 4
03:07:47.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.988274 28180 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Bootstrap starting.
03:07:47.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:47 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001662, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.62@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:07:47.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.996637 28180 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Neither blocks nor log segments found. Creating new log.
03:07:48.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:47.999727 28180 log.cc:826] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Log is configured to *not* fsync() on all Append() calls
03:07:48.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.008576 28180 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: No bootstrap required, opened a new log
03:07:48.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.018358 28008 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Bootstrap starting.
03:07:48.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.024698 28008 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Neither blocks nor log segments found. Creating new log.
03:07:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.028059 28008 log.cc:826] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Log is configured to *not* fsync() on all Append() calls
03:07:48.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.034017 28008 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: No bootstrap required, opened a new log
03:07:48.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.036434 28180 raft_consensus.cc:359] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:07:48.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.037210 28180 raft_consensus.cc:385] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:48.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.037551 28180 raft_consensus.cc:740] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 080b9f8181db40ccb2df34a96fcb4738, State: Initialized, Role: FOLLOWER
03:07:48.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.038770 28180 consensus_queue.cc:260] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:07:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.044142 28192 sys_catalog.cc:455] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:07:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.045045 28192 sys_catalog.cc:458] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: This master's current role is: FOLLOWER
03:07:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.045748 28180 sys_catalog.cc:565] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: configured and running, proceeding with master startup.
03:07:48.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.057826 28008 raft_consensus.cc:359] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:07:48.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.058755 28008 raft_consensus.cc:385] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:48.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.059083 28008 raft_consensus.cc:740] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 040a41d80f044969aaa60adb75a339c5, State: Initialized, Role: FOLLOWER
03:07:48.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.060245 28008 consensus_queue.cc:260] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:07:48.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.069347 28200 sys_catalog.cc:455] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:07:48.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.072049 28200 sys_catalog.cc:458] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: This master's current role is: FOLLOWER
03:07:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.073506 28008 sys_catalog.cc:565] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: configured and running, proceeding with master startup.
03:07:48.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.075866 28204 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:07:48.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.082892 28192 raft_consensus.cc:493] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:07:48.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.083472 28192 raft_consensus.cc:515] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:07:48.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.088654 28192 leader_election.cc:290] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407), 4d360522e835469d8de898131ddaaafc (127.27.65.61:42281)
03:07:48.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.089223 27983 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "080b9f8181db40ccb2df34a96fcb4738" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "040a41d80f044969aaa60adb75a339c5" is_pre_election: true
03:07:48.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.090875 27983 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 080b9f8181db40ccb2df34a96fcb4738 in term 0.
03:07:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.097496 28129 leader_election.cc:304] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 040a41d80f044969aaa60adb75a339c5, 080b9f8181db40ccb2df34a96fcb4738; no voters:
03:07:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.098691 28192 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:07:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.099098 28192 raft_consensus.cc:493] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:07:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.099524 28192 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 0 FOLLOWER]: Advancing to term 1
03:07:48.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.107577 28192 raft_consensus.cc:515] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:07:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.109259 28095 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } attempt: 7
03:07:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.109558 28192 leader_election.cc:290] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [CANDIDATE]: Term 1 election: Requested vote from peers 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407), 4d360522e835469d8de898131ddaaafc (127.27.65.61:42281)
03:07:48.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.110472 27983 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "080b9f8181db40ccb2df34a96fcb4738" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "040a41d80f044969aaa60adb75a339c5"
03:07:48.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.111107 27983 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [term 0 FOLLOWER]: Advancing to term 1
03:07:48.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:48.115851 27962 tablet.cc:2378] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:48.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.120078 27983 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 080b9f8181db40ccb2df34a96fcb4738 in term 1.
03:07:48.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.121443 28129 leader_election.cc:304] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 040a41d80f044969aaa60adb75a339c5, 080b9f8181db40ccb2df34a96fcb4738; no voters:
03:07:48.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.125872 28192 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 1 FOLLOWER]: Leader election won for term 1
03:07:48.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.131945 28192 raft_consensus.cc:697] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 1 LEADER]: Becoming Leader. State: Replica: 080b9f8181db40ccb2df34a96fcb4738, State: Running, Role: LEADER
03:07:48.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.133023 28192 consensus_queue.cc:237] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:07:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.143851 28212 sys_catalog.cc:455] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 080b9f8181db40ccb2df34a96fcb4738. Latest consensus state: current_term: 1 leader_uuid: "080b9f8181db40ccb2df34a96fcb4738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:07:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.145318 28212 sys_catalog.cc:458] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: This master's current role is: LEADER
03:07:48.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.079845 28105 master_runner.cc:186] Error getting master registration for 127.27.65.62:41407: OK, instance_id { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" instance_seqno: 1770001662934218 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:48.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.161510 28105 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:48.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.176198 28218 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:07:48.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:48 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001665, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.61@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:07:48.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:48.209074 28219 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:07:48.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:48.209461 28219 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:07:48.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.212148 28218 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:07:48.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.228766 28095 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Bootstrap starting.
03:07:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.192170 27938 master_runner.cc:186] Error getting master registration for 127.27.65.61:42281: OK, instance_id { permanent_uuid: "4d360522e835469d8de898131ddaaafc" instance_seqno: 1770001665685789 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:07:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.232754 27938 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:07:48.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.246227 28095 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Neither blocks nor log segments found. Creating new log.
03:07:48.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:48.251925 28134 tablet.cc:2378] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:48.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.255712 28095 log.cc:826] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Log is configured to *not* fsync() on all Append() calls
03:07:48.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.275400 28095 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: No bootstrap required, opened a new log
03:07:48.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.282284 27983 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [term 1 FOLLOWER]: Refusing update from remote peer 080b9f8181db40ccb2df34a96fcb4738: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:48.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.285212 28212 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [LEADER]: Connected to new peer: Peer: permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:07:48.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.335413 28095 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:07:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.336707 28095 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:07:48.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.337182 28095 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4d360522e835469d8de898131ddaaafc, State: Initialized, Role: FOLLOWER
03:07:48.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.339542 28095 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:07:48.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.352286 28095 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: configured and running, proceeding with master startup.
03:07:48.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.354782 28069 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "080b9f8181db40ccb2df34a96fcb4738" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4d360522e835469d8de898131ddaaafc"
03:07:48.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.352460 28070 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "080b9f8181db40ccb2df34a96fcb4738" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4d360522e835469d8de898131ddaaafc" is_pre_election: true
03:07:48.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.354230 28068 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 0 FOLLOWER]: Advancing to term 1
03:07:48.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.354059 28225 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:07:48.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.360960 28225 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: This master's current role is: FOLLOWER
03:07:48.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.373162 28068 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 1 FOLLOWER]: Refusing update from remote peer 080b9f8181db40ccb2df34a96fcb4738: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:07:48.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.375577 28212 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
03:07:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:48.381868 28048 tablet.cc:2378] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:07:48.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.409442 28200 sys_catalog.cc:455] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 080b9f8181db40ccb2df34a96fcb4738. Latest consensus state: current_term: 1 leader_uuid: "080b9f8181db40ccb2df34a96fcb4738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:07:48.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.410249 28200 sys_catalog.cc:458] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: This master's current role is: FOLLOWER
03:07:48.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.427423 28222 mvcc.cc:204] Tried to move back new op lower bound from 7249926833218424832 to 7249926832685580288. Current Snapshot: MvccSnapshot[applied={T|T < 7249926833218424832}]
03:07:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.433961 28200 sys_catalog.cc:455] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "080b9f8181db40ccb2df34a96fcb4738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:07:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.434718 28200 sys_catalog.cc:458] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: This master's current role is: FOLLOWER
03:07:48.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.450976 28192 sys_catalog.cc:455] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "080b9f8181db40ccb2df34a96fcb4738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:07:48.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.451797 28192 sys_catalog.cc:458] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: This master's current role is: LEADER
03:07:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.460477 28225 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 080b9f8181db40ccb2df34a96fcb4738. Latest consensus state: current_term: 1 leader_uuid: "080b9f8181db40ccb2df34a96fcb4738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:07:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.461172 28218 catalog_manager.cc:1357] Generated new cluster ID: 7dd52b2ccf104be08a86cdd0bbc27a05
03:07:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.461229 28225 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: This master's current role is: FOLLOWER
03:07:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.461484 28218 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:07:48.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.464205 28212 sys_catalog.cc:455] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "080b9f8181db40ccb2df34a96fcb4738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:07:48.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.464949 28212 sys_catalog.cc:458] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: This master's current role is: LEADER
03:07:48.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.470084 28230 mvcc.cc:204] Tried to move back new op lower bound from 7249926833218424832 to 7249926832685580288. Current Snapshot: MvccSnapshot[applied={T|T < 7249926833218424832}]
03:07:48.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.487617 28225 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "080b9f8181db40ccb2df34a96fcb4738" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:07:48.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.488303 28225 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: This master's current role is: FOLLOWER
03:07:48.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.524142 28218 catalog_manager.cc:1380] Generated new certificate authority record
03:07:48.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.526434 28218 catalog_manager.cc:1514] Loading token signing keys...
03:07:48.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.574281 28218 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Generated new TSK 0
03:07:48.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.576800 28218 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:07:48.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.588395 28242 catalog_manager.cc:1269] Loaded cluster ID: 7dd52b2ccf104be08a86cdd0bbc27a05
03:07:48.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.588721 28242 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: loading cluster ID for follower catalog manager: success
03:07:48.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.599056 28242 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: acquiring CA information for follower catalog manager: success
03:07:48.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.606824 28242 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:48.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.65.1@KRBTEST.COM; defaulting to no policy
03:07:48.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:48.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.65.1@KRBTEST.COM" created.
03:07:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:48.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:48.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.65.1@KRBTEST.COM; defaulting to no policy
03:07:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:48.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.65.1@KRBTEST.COM" created.
03:07:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:48.725462 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/wal
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/data
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/logs
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/data/info.pb
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:48.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.1:0
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.65.1
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.1
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.1
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:48.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:48.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:48.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:48.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:48.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:49.215339 28219 catalog_manager.cc:1269] Loaded cluster ID: 7dd52b2ccf104be08a86cdd0bbc27a05
03:07:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:49.215652 28219 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: loading cluster ID for follower catalog manager: success
03:07:49.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:49.223556 28219 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: acquiring CA information for follower catalog manager: success
03:07:49.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:49.239490 28219 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:07:49.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:49.563090 28248 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:49.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:49.564895 28248 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:49.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:49.566129 28248 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:49.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:49.630535 28248 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:49.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:49.632772 28248 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.65.1
03:07:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:49.704020 28248 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/data
03:07:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/wal
03:07:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.1
03:07:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.1:0
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/data/info.pb
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.1
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.65.1
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/logs
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:49.719750 28248 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:49.721653 28248 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:49.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:49.756916 28254 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:52.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:49.764045 28255 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:51.154199 28253 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 28248
03:07:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:52.474834 28253 kernel_stack_watchdog.cc:198] Thread 28248 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 390ms:
03:07:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:52.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:52.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:52.475734 28248 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.714s user 0.707s sys 1.106s
03:07:52.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:52.477850 28248 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.717s user 0.707s sys 1.106s
03:07:52.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:52.507848 28261 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:52.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.509886 28248 server_base.cc:1060] running on GCE node
03:07:52.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.511749 28248 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:52.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.519999 28248 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:52.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.531056 28248 hybrid_clock.cc:648] HybridClock initialized: now 1770001672530973 us; error 56 us; skew 500 ppm
03:07:52.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.532478 28248 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:52.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:52 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001672, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:52.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.542059 28248 init.cc:377] Logged in from keytab as kudu/127.27.65.1@KRBTEST.COM (short username kudu)
03:07:52.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.572041 28248 webserver.cc:492] Webserver started at http://127.27.65.1:34711/ using document root <none> and password file <none>
03:07:52.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.573613 28248 fs_manager.cc:362] Metadata directory not provided
03:07:52.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.573904 28248 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:52.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.574465 28248 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.579566 28248 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/data/instance:
03:07:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c32ac7effcab4a3b92bf2b6f555a0c1e"
03:07:52.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:52 on dist-test-slave-0wq0"
03:07:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.581173 28248 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/wal/instance:
03:07:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c32ac7effcab4a3b92bf2b6f555a0c1e"
03:07:52.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:52 on dist-test-slave-0wq0"
03:07:52.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.590505 28248 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.002s
03:07:52.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.597175 28267 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:52.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.598523 28248 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
03:07:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.598948 28248 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/wal
03:07:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c32ac7effcab4a3b92bf2b6f555a0c1e"
03:07:52.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:52 on dist-test-slave-0wq0"
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.599328 28248 fs_report.cc:389] FS layout report
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/wal
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/wal
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/data/data
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:52.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:52.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.694208 28248 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:52.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.707530 28248 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:52.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.708501 28248 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:52.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:52 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001672, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:52.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.725432 27952 init.cc:295] Successfully reacquired a new kerberos TGT
03:07:52.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.731722 28248 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:52.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.752952 28248 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:52.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.753276 28248 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:52.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.753566 28248 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:52.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.753773 28248 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:52.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.978138 28248 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.1:35391
03:07:52.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.978717 28383 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.1:35391 every 8 connection(s)
03:07:52.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.981986 28248 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/data/info.pb
03:07:52.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.985808 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 28248
03:07:52.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:52.986328 27908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-0/wal/instance
03:07:53.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.65.2@KRBTEST.COM; defaulting to no policy
03:07:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:53 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001672, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.1@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:07:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:53.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.65.2@KRBTEST.COM" created.
03:07:53.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:53 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001672, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.1@KRBTEST.COM for kudu/127.27.65.62@KRBTEST.COM
03:07:53.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:53 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001672, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.1@KRBTEST.COM for kudu/127.27.65.61@KRBTEST.COM
03:07:53.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.082968 28384 heartbeater.cc:344] Connected to a master server at 127.27.65.60:38795
03:07:53.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.083597 28384 heartbeater.cc:461] Registering TS with master...
03:07:53.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.085151 28384 heartbeater.cc:507] Master 127.27.65.60:38795 requested a full tablet report, sending...
03:07:53.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.091501 28385 heartbeater.cc:344] Connected to a master server at 127.27.65.61:42281
03:07:53.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.093015 28385 heartbeater.cc:461] Registering TS with master...
03:07:53.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.094581 28145 ts_manager.cc:194] Registered new tserver with Master: c32ac7effcab4a3b92bf2b6f555a0c1e (127.27.65.1:35391)
03:07:53.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.095741 28385 heartbeater.cc:507] Master 127.27.65.61:42281 requested a full tablet report, sending...
03:07:53.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.098433 28145 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.27.65.1@KRBTEST.COM'} at 127.27.65.1:48193
03:07:53.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.099472 28060 ts_manager.cc:194] Registered new tserver with Master: c32ac7effcab4a3b92bf2b6f555a0c1e (127.27.65.1:35391)
03:07:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.108832 28389 heartbeater.cc:344] Connected to a master server at 127.27.65.62:41407
03:07:53.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.109236 28389 heartbeater.cc:461] Registering TS with master...
03:07:53.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.110005 28389 heartbeater.cc:507] Master 127.27.65.62:41407 requested a full tablet report, sending...
03:07:53.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:53.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:53.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:53.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:53.111248 27908 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.27.65.2: real 0.111s user 0.003s sys 0.014s
03:07:53.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.115463 27973 ts_manager.cc:194] Registered new tserver with Master: c32ac7effcab4a3b92bf2b6f555a0c1e (127.27.65.1:35391)
03:07:53.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.65.2@KRBTEST.COM; defaulting to no policy
03:07:53.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:53.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.65.2@KRBTEST.COM" created.
03:07:53.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:53.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:53.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.160907 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/wal
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/data
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/logs
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/data/info.pb
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:53.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.2:0
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.65.2
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.2
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.2
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:53.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:53.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:53.537878 28399 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:53.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:53.538354 28399 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:53.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:53.539022 28399 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:53.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:53.574218 28399 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:53.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:53.575241 28399 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.65.2
03:07:53.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.613425 28399 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/data
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/wal
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.2
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.2:0
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:53.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/data/info.pb
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.2
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.65.2
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/logs
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:53.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.614992 28399 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:53.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:53.616006 28399 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:53.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:53.629109 28407 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:54.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:54.104077 28384 heartbeater.cc:499] Master 127.27.65.60:38795 was elected leader, sending a full tablet report...
03:07:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:55.032095 28406 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 28399
03:07:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:55.494509 28406 kernel_stack_watchdog.cc:198] Thread 28399 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
03:07:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:07:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:07:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:07:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:07:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:55.494764 28399 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.865s user 0.000s sys 0.002s
03:07:55.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:55.495229 28399 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.865s user 0.000s sys 0.003s
03:07:55.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:53.630889 28408 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:55.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:55.496197 28409 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1863 milliseconds
03:07:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:55.497241 28410 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:55.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.497108 28399 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:55.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.503463 28399 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:55.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.507018 28399 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:55.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.508567 28399 hybrid_clock.cc:648] HybridClock initialized: now 1770001675508519 us; error 51 us; skew 500 ppm
03:07:55.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.509657 28399 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:55.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:55 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001675, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:55.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:55 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001675, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:55.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.518285 28038 init.cc:295] Successfully reacquired a new kerberos TGT
03:07:55.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.518844 28399 init.cc:377] Logged in from keytab as kudu/127.27.65.2@KRBTEST.COM (short username kudu)
03:07:55.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.526561 28399 webserver.cc:492] Webserver started at http://127.27.65.2:39377/ using document root <none> and password file <none>
03:07:55.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.527833 28399 fs_manager.cc:362] Metadata directory not provided
03:07:55.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.528131 28399 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:55.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.528674 28399 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:55.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.535347 28399 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/data/instance:
03:07:55.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c187dbbc057548cdb1c948a1f6d94b64"
03:07:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:55 on dist-test-slave-0wq0"
03:07:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.537185 28399 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/wal/instance:
03:07:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c187dbbc057548cdb1c948a1f6d94b64"
03:07:55.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:55 on dist-test-slave-0wq0"
03:07:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.547246 28399 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.004s sys 0.007s
03:07:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.553779 28419 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.555150 28399 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.001s
03:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.555524 28399 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/wal
03:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c187dbbc057548cdb1c948a1f6d94b64"
03:07:55.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:55 on dist-test-slave-0wq0"
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.555855 28399 fs_report.cc:389] FS layout report
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/wal
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/wal
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/data/data
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:55.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:55.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.625391 28399 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.631757 28399 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:55.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.632197 28399 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:55.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.635085 28399 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.640041 28399 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.640275 28399 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.640547 28399 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.640784 28399 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:55.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.832814 28399 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.2:35089
03:07:55.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.832974 28532 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.2:35089 every 8 connection(s)
03:07:55.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.835523 28399 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/data/info.pb
03:07:55.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.839329 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 28399
03:07:55.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.839908 27908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-1/wal/instance
03:07:55.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:55 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001675, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.2@KRBTEST.COM for kudu/127.27.65.61@KRBTEST.COM
03:07:55.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:55 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001675, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.2@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:07:55.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.65.3@KRBTEST.COM; defaulting to no policy
03:07:55.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:55 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001675, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.2@KRBTEST.COM for kudu/127.27.65.62@KRBTEST.COM
03:07:55.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.936367 28534 heartbeater.cc:344] Connected to a master server at 127.27.65.61:42281
03:07:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.936977 28534 heartbeater.cc:461] Registering TS with master...
03:07:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:55.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.65.3@KRBTEST.COM" created.
03:07:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.937841 28533 heartbeater.cc:344] Connected to a master server at 127.27.65.60:38795
03:07:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.938220 28533 heartbeater.cc:461] Registering TS with master...
03:07:55.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.938510 28534 heartbeater.cc:507] Master 127.27.65.61:42281 requested a full tablet report, sending...
03:07:55.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.939455 28533 heartbeater.cc:507] Master 127.27.65.60:38795 requested a full tablet report, sending...
03:07:55.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.941890 28145 ts_manager.cc:194] Registered new tserver with Master: c187dbbc057548cdb1c948a1f6d94b64 (127.27.65.2:35089)
03:07:55.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.942873 28060 ts_manager.cc:194] Registered new tserver with Master: c187dbbc057548cdb1c948a1f6d94b64 (127.27.65.2:35089)
03:07:55.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.943982 28145 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.27.65.2@KRBTEST.COM'} at 127.27.65.2:35509
03:07:55.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.949657 28535 heartbeater.cc:344] Connected to a master server at 127.27.65.62:41407
03:07:55.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.950086 28535 heartbeater.cc:461] Registering TS with master...
03:07:55.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.950807 28535 heartbeater.cc:507] Master 127.27.65.62:41407 requested a full tablet report, sending...
03:07:55.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:55.953675 27973 ts_manager.cc:194] Registered new tserver with Master: c187dbbc057548cdb1c948a1f6d94b64 (127.27.65.2:35089)
03:07:55.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:55.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:55.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.65.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:55.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:55.967888 27908 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.27.65.3: real 0.106s user 0.010s sys 0.007s
03:07:55.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.65.3@KRBTEST.COM; defaulting to no policy
03:07:55.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:55.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.65.3@KRBTEST.COM" created.
03:07:56.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:07:56.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:56.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.65.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab.
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:56.018810 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/wal
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/data
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/logs
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/data/info.pb
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:56.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.3:0
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.65.3
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.3
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.3
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:56.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:56.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:56.361153 28547 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:56.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:56.361631 28547 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:56.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:56.362251 28547 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:56.396406 28547 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:56.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:56.397316 28547 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.65.3
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:56.435714 28547 tablet_server_runner.cc:78] Tablet server non-default flags:
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/data
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/wal
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.3
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.3:0
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/data/info.pb
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.3
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:56.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.65.3
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/logs
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:56.436940 28547 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:56.438134 28547 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:56.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:56.452158 28554 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:56.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:56.948681 28533 heartbeater.cc:499] Master 127.27.65.60:38795 was elected leader, sending a full tablet report...
03:07:57.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:57 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001677, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:57.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.650106 28124 init.cc:295] Successfully reacquired a new kerberos TGT
03:07:57.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:56.458745 28557 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:57.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:56.453627 28555 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:57.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:57.826552 28556 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1368 milliseconds
03:07:57.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.826717 28547 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:07:57.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.828121 28547 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:07:57.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.830529 28547 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:07:57.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.831903 28547 hybrid_clock.cc:648] HybridClock initialized: now 1770001677831858 us; error 59 us; skew 500 ppm
03:07:57.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.833039 28547 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:07:57.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:57 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001677, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:57.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.840018 28547 init.cc:377] Logged in from keytab as kudu/127.27.65.3@KRBTEST.COM (short username kudu)
03:07:57.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.846781 28547 webserver.cc:492] Webserver started at http://127.27.65.3:43589/ using document root <none> and password file <none>
03:07:57.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.847903 28547 fs_manager.cc:362] Metadata directory not provided
03:07:57.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.848150 28547 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:07:57.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.848662 28547 server_base.cc:908] This appears to be a new deployment of Kudu; creating new FS layout
03:07:57.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.853013 28547 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/data/instance:
03:07:57.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61749f6f946d469197f1bd9e080f9108"
03:07:57.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:57 on dist-test-slave-0wq0"
03:07:57.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.854156 28547 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/wal/instance:
03:07:57.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61749f6f946d469197f1bd9e080f9108"
03:07:57.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:57 on dist-test-slave-0wq0"
03:07:57.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.863754 28547 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.009s sys 0.001s
03:07:57.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.869867 28566 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:07:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.871161 28547 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
03:07:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.871541 28547 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/wal
03:07:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "61749f6f946d469197f1bd9e080f9108"
03:07:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:57 on dist-test-slave-0wq0"
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.871901 28547 fs_report.cc:389] FS layout report
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/wal
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/wal
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/data/data
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:07:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:07:57.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.939649 28547 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:07:57.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.949975 28547 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:07:57.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.950410 28547 kserver.cc:163] Server-wide thread pool size limit: 3276
03:07:57.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.953219 28547 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:07:57.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.958572 28547 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:07:57.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.958894 28547 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:07:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.959211 28547 ts_tablet_manager.cc:616] Registered 0 tablets
03:07:57.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:57.959419 28547 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:07:58.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.191851 28547 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.3:37113
03:07:58.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.191973 28679 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.3:37113 every 8 connection(s)
03:07:58.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.194701 28547 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/data/info.pb
03:07:58.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.196810 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 28547
03:07:58.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.197389 27908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/ts-2/wal/instance
03:07:58.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:58 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001677, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.3@KRBTEST.COM for kudu/127.27.65.61@KRBTEST.COM
03:07:58.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:58 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001677, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.3@KRBTEST.COM for kudu/127.27.65.62@KRBTEST.COM
03:07:58.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.308319 28681 heartbeater.cc:344] Connected to a master server at 127.27.65.61:42281
03:07:58.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.309216 28681 heartbeater.cc:461] Registering TS with master...
03:07:58.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:58 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001677, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.3@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:07:58.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.312640 28681 heartbeater.cc:507] Master 127.27.65.61:42281 requested a full tablet report, sending...
03:07:58.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.317999 28060 ts_manager.cc:194] Registered new tserver with Master: 61749f6f946d469197f1bd9e080f9108 (127.27.65.3:37113)
03:07:58.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.327841 28684 heartbeater.cc:344] Connected to a master server at 127.27.65.62:41407
03:07:58.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.328440 28684 heartbeater.cc:461] Registering TS with master...
03:07:58.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.330621 28684 heartbeater.cc:507] Master 127.27.65.62:41407 requested a full tablet report, sending...
03:07:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.335870 27972 ts_manager.cc:194] Registered new tserver with Master: 61749f6f946d469197f1bd9e080f9108 (127.27.65.3:37113)
03:07:58.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.349016 28680 heartbeater.cc:344] Connected to a master server at 127.27.65.60:38795
03:07:58.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.349383 28680 heartbeater.cc:461] Registering TS with master...
03:07:58.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.350739 28680 heartbeater.cc:507] Master 127.27.65.60:38795 requested a full tablet report, sending...
03:07:58.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.354943 28145 ts_manager.cc:194] Registered new tserver with Master: 61749f6f946d469197f1bd9e080f9108 (127.27.65.3:37113)
03:07:58.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.357870 27908 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
03:07:58.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.358348 28145 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.27.65.3@KRBTEST.COM'} at 127.27.65.3:53295
03:07:58.372 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:58.374 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:07:58.384 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:07:58.385 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:07:58.388 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.65.62"
port: 41407
}
bound_http_address {
host: "127.27.65.62"
port: 36607
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.65.61"
port: 42281
}
bound_http_address {
host: "127.27.65.61"
port: 36875
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.65.60"
port: 38795
}
bound_http_address {
host: "127.27.65.60"
port: 41997
}
}
}
03:07:58.388 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:07:58.390 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.65.1"
port: 35391
}
bound_http_address {
host: "127.27.65.1"
port: 34711
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.65.2"
port: 35089
}
bound_http_address {
host: "127.27.65.2"
port: 39377
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.65.3"
port: 37113
}
bound_http_address {
host: "127.27.65.3"
port: 43589
}
}
}
03:07:58.390 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:07:58.392 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:07:58.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:58 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001678, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:58.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:58.436 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:58.439 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc
03:07:58.445 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:58.446 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc
03:07:58.448 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:58.452 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
03:07:58.453 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:07:58.454 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:07:58.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:07:58 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001678, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:07:58.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:07:58.490 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:58.490 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc
03:07:58.492 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:07:58.492 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.65.61:42281
03:07:58.494 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:07:58.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.495162 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 28017
03:07:58.524 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:58.525 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.65.60:38795
03:07:58.525 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:07:58.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.525517 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 28105
03:07:58.553 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:58.553 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.65.62:41407
03:07:58.554 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:07:58.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.554440 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 27938
03:07:58.584 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:07:58.584 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.65.61:42281
03:07:58.588 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:58.589622 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/logs
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/info.pb
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:07:58.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.61:42281
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.61
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36875
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.61
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:07:58.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:07:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:58.920976 28696 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:07:58.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:58.921526 28696 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:07:58.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:58.922140 28696 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:07:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:58.956492 28696 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:07:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:58.956740 28696 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:07:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:58.957005 28696 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:07:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:58.957190 28696 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:07:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:58.957372 28696 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:07:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:58.957640 28696 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:07:58.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:58.957803 28696 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42881/jwks.json
03:07:58.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:58.967715 28535 heartbeater.cc:646] Failed to heartbeat to 127.27.65.62:41407 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111)
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:59.000726 28696 master_runner.cc:386] Master server non-default flags:
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.61
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.61:42281
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/info.pb
03:07:59.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.61
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36875
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/logs
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:07:59.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:59.001989 28696 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:07:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:59.003130 28696 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:07:59.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:59.014879 28704 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:07:59.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:59.165158 28385 heartbeater.cc:646] Failed to heartbeat to 127.27.65.61:42281 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.27.65.61:42281: connect: Connection refused (error 111)
03:07:59.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:59.328027 28681 heartbeater.cc:646] Failed to heartbeat to 127.27.65.61:42281 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.27.65.61:42281: connect: Connection refused (error 111)
03:07:59.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:07:59.362816 28680 heartbeater.cc:499] Master 127.27.65.60:38795 was elected leader, sending a full tablet report...
03:08:00.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:07:59.016086 28705 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.367049 28706 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1349 milliseconds
03:08:00.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.367465 28707 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.368026 28696 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:00.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.371742 28696 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:00.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.374328 28696 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.375815 28696 hybrid_clock.cc:648] HybridClock initialized: now 1770001680375751 us; error 71 us; skew 500 ppm
03:08:00.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.376685 28696 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:00.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:00 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001680, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:00.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.383667 28696 init.cc:377] Logged in from keytab as kudu/127.27.65.61@KRBTEST.COM (short username kudu)
03:08:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.389720 28696 webserver.cc:492] Webserver started at http://127.27.65.61:36875/ using document root <none> and password file <none>
03:08:00.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.390733 28696 fs_manager.cc:362] Metadata directory not provided
03:08:00.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.391004 28696 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:00.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.399220 28696 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.003s sys 0.005s
03:08:00.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.404047 28725 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:00.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.405236 28696 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
03:08:00.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.405593 28696 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:08:00.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d360522e835469d8de898131ddaaafc"
03:08:00.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:45 on dist-test-slave-0wq0"
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.407704 28696 fs_report.cc:389] FS layout report
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/data
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:00.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:00.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.467574 28696 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.473127 28696 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:00.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.473532 28696 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.562878 28696 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.61:42281
03:08:00.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.563125 28776 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.61:42281 every 8 connection(s)
03:08:00.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.565719 28696 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/info.pb
03:08:00.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.566987 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 28696
03:08:00.567 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:00.568 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.65.60:38795
03:08:00.568 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:08:00.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.570382 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:08:00.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/logs
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/info.pb
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:00.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.60:38795
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.60
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41997
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.60
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:00.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:00.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.580958 28777 sys_catalog.cc:263] Verifying existing consensus state
03:08:00.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.587709 28777 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Bootstrap starting.
03:08:00.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.629796 28777 log.cc:826] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Log is configured to *not* fsync() on all Append() calls
03:08:00.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.646072 28777 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:00.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.646840 28777 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Bootstrap complete.
03:08:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.670895 28777 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:00.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.673305 28777 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4d360522e835469d8de898131ddaaafc, State: Initialized, Role: FOLLOWER
03:08:00.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.674307 28777 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:00.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.677817 28781 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:00.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.678830 28781 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: This master's current role is: FOLLOWER
03:08:00.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.682139 28777 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: configured and running, proceeding with master startup.
03:08:00.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.724231 28792 catalog_manager.cc:1269] Loaded cluster ID: 7dd52b2ccf104be08a86cdd0bbc27a05
03:08:00.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.724948 28792 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: loading cluster ID for follower catalog manager: success
03:08:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.735700 28792 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: acquiring CA information for follower catalog manager: success
03:08:00.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.741382 28792 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:00.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.713685 28696 master_runner.cc:186] Error getting master registration for 127.27.65.62:41407: Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111),
03:08:00.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:00.752092 28696 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:00.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.945967 28779 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:00.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.946656 28779 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:00.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.947522 28779 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:00.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.980954 28779 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:00.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.981277 28779 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.981694 28779 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.982044 28779 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:00.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.982359 28779 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.982707 28779 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:00.983013 28779 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42881/jwks.json
03:08:01.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.030016 28534 heartbeater.cc:344] Connected to a master server at 127.27.65.61:42281
03:08:01.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.036677 28742 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c187dbbc057548cdb1c948a1f6d94b64" instance_seqno: 1770001675792422) as {username='kudu', principal='kudu/127.27.65.2@KRBTEST.COM'} at 127.27.65.2:51151; Asking this server to re-register.
03:08:01.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.039124 28534 heartbeater.cc:461] Registering TS with master...
03:08:01.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.042681 28742 ts_manager.cc:194] Registered new tserver with Master: c187dbbc057548cdb1c948a1f6d94b64 (127.27.65.2:35089)
03:08:01.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.043421 28779 master_runner.cc:386] Master server non-default flags:
03:08:01.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:01.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.60
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.60:38795
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/info.pb
03:08:01.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.60
03:08:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41997
03:08:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:01.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/logs
03:08:01.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:01.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:08:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:08:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.045178 28779 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:01.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.046576 28779 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:01.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:01.062139 28804 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:01.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.218555 28385 heartbeater.cc:344] Connected to a master server at 127.27.65.61:42281
03:08:01.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.223173 28742 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c32ac7effcab4a3b92bf2b6f555a0c1e" instance_seqno: 1770001672936085) as {username='kudu', principal='kudu/127.27.65.1@KRBTEST.COM'} at 127.27.65.1:47765; Asking this server to re-register.
03:08:01.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.227386 28385 heartbeater.cc:461] Registering TS with master...
03:08:01.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.232247 28742 ts_manager.cc:194] Registered new tserver with Master: c32ac7effcab4a3b92bf2b6f555a0c1e (127.27.65.1:35391)
03:08:01.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.407621 28681 heartbeater.cc:344] Connected to a master server at 127.27.65.61:42281
03:08:01.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.413329 28742 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "61749f6f946d469197f1bd9e080f9108" instance_seqno: 1770001678109585) as {username='kudu', principal='kudu/127.27.65.3@KRBTEST.COM'} at 127.27.65.3:49987; Asking this server to re-register.
03:08:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.415710 28681 heartbeater.cc:461] Registering TS with master...
03:08:01.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.420611 28742 ts_manager.cc:194] Registered new tserver with Master: 61749f6f946d469197f1bd9e080f9108 (127.27.65.3:37113)
03:08:01.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.874580 28814 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:01.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.876072 28814 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:01.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.933037 28814 leader_election.cc:290] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407), 080b9f8181db40ccb2df34a96fcb4738 (127.27.65.60:38795)
03:08:01.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:01.941532 28726 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111)
03:08:01.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:01.963470 28728 leader_election.cc:336] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 080b9f8181db40ccb2df34a96fcb4738 (127.27.65.60:38795): Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:08:01.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:01.968998 28726 leader_election.cc:336] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407): Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111)
03:08:01.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.970027 28726 leader_election.cc:304] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4d360522e835469d8de898131ddaaafc; no voters: 040a41d80f044969aaa60adb75a339c5, 080b9f8181db40ccb2df34a96fcb4738
03:08:01.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:01.971369 28814 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
03:08:02.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:02 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001682, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:02.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.559612 28263 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:02.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:02.465067 28802 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 28779
03:08:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:02.755698 28802 kernel_stack_watchdog.cc:198] Thread 28779 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
03:08:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:02.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:01.063455 28803 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:02.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:02.757392 28779 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.694s user 0.488s sys 0.847s
03:08:02.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:02.757786 28779 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.695s user 0.488s sys 0.847s
03:08:02.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:02.758224 28807 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:02.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:02.760782 28805 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1694 milliseconds
03:08:02.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.760802 28779 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:02.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.762136 28779 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.765518 28779 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:02.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.767017 28779 hybrid_clock.cc:648] HybridClock initialized: now 1770001682766959 us; error 73 us; skew 500 ppm
03:08:02.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.767825 28779 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:02.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:02 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001682, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:02.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.774363 28779 init.cc:377] Logged in from keytab as kudu/127.27.65.60@KRBTEST.COM (short username kudu)
03:08:02.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.780313 28779 webserver.cc:492] Webserver started at http://127.27.65.60:41997/ using document root <none> and password file <none>
03:08:02.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.781246 28779 fs_manager.cc:362] Metadata directory not provided
03:08:02.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.781553 28779 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:02.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.790328 28779 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.001s
03:08:02.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.796089 28829 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:02.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.797565 28779 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
03:08:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.798089 28779 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:08:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "080b9f8181db40ccb2df34a96fcb4738"
03:08:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:47 on dist-test-slave-0wq0"
03:08:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.800320 28779 fs_report.cc:389] FS layout report
03:08:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:02.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/data
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:02.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:02.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.876395 28779 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:02.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.882601 28779 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:02.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.883251 28779 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:02.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.978057 28779 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.60:38795
03:08:02.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.978134 28880 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.60:38795 every 8 connection(s)
03:08:02.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.981280 28779 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/info.pb
03:08:02.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.982995 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 28779
03:08:02.984 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:02.984 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.65.62:41407
03:08:02.985 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.986972 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/logs
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/info.pb
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.62:41407
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.62
03:08:02.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36607
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.62
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:02.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:02.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:02.999394 28881 sys_catalog.cc:263] Verifying existing consensus state
03:08:03.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.007741 28881 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Bootstrap starting.
03:08:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.069764 28881 log.cc:826] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Log is configured to *not* fsync() on all Append() calls
03:08:03.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.093583 28533 heartbeater.cc:344] Connected to a master server at 127.27.65.60:38795
03:08:03.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.095268 28881 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:03.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.097963 28881 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Bootstrap complete.
03:08:03.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.128698 28881 raft_consensus.cc:359] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:03.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.132176 28881 raft_consensus.cc:740] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 080b9f8181db40ccb2df34a96fcb4738, State: Initialized, Role: FOLLOWER
03:08:03.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.133291 28881 consensus_queue.cc:260] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:03.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.142508 28887 sys_catalog.cc:455] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:03.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.143474 28887 sys_catalog.cc:458] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: This master's current role is: FOLLOWER
03:08:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.145848 28881 sys_catalog.cc:565] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: configured and running, proceeding with master startup.
03:08:03.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.186902 28898 catalog_manager.cc:1269] Loaded cluster ID: 7dd52b2ccf104be08a86cdd0bbc27a05
03:08:03.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.203034 28898 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: loading cluster ID for follower catalog manager: success
03:08:03.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.220988 28898 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: acquiring CA information for follower catalog manager: success
03:08:03.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.243748 28898 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:03.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.266741 28384 heartbeater.cc:344] Connected to a master server at 127.27.65.60:38795
03:08:03.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.217541 28779 master_runner.cc:186] Error getting master registration for 127.27.65.62:41407: Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111),
03:08:03.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.269345 28779 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.273407 28846 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c32ac7effcab4a3b92bf2b6f555a0c1e" instance_seqno: 1770001672936085) as {username='kudu', principal='kudu/127.27.65.1@KRBTEST.COM'} at 127.27.65.1:54697; Asking this server to re-register.
03:08:03.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.274765 28384 heartbeater.cc:461] Registering TS with master...
03:08:03.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.278244 28846 ts_manager.cc:194] Registered new tserver with Master: c32ac7effcab4a3b92bf2b6f555a0c1e (127.27.65.1:35391)
03:08:03.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.448465 28680 heartbeater.cc:344] Connected to a master server at 127.27.65.60:38795
03:08:03.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.449007 28680 heartbeater.cc:499] Master 127.27.65.60:38795 was elected leader, sending a full tablet report...
03:08:03.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.454742 28846 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "61749f6f946d469197f1bd9e080f9108" instance_seqno: 1770001678109585) as {username='kudu', principal='kudu/127.27.65.3@KRBTEST.COM'} at 127.27.65.3:57911; Asking this server to re-register.
03:08:03.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.456900 28680 heartbeater.cc:461] Registering TS with master...
03:08:03.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.463799 28846 ts_manager.cc:194] Registered new tserver with Master: 61749f6f946d469197f1bd9e080f9108 (127.27.65.3:37113)
03:08:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.532006 28883 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:03.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.532939 28883 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:03.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.534727 28883 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:03.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.599431 28883 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:03.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.599773 28883 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.600193 28883 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:03.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.600528 28883 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.600854 28883 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.601158 28883 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:03.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.601395 28883 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42881/jwks.json
03:08:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.669570 28883 master_runner.cc:386] Master server non-default flags:
03:08:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:08:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:03.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.62
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.62:41407
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/info.pb
03:08:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.62
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36607
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/logs
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.671296 28883 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:03.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:03.672636 28883 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:03.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.687120 28913 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:04.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.055804 28920 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:04.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.058303 28920 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:04.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.061307 28920 leader_election.cc:290] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407), 080b9f8181db40ccb2df34a96fcb4738 (127.27.65.60:38795)
03:08:04.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:04.160797 28726 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.27.65.62:41407, user_credentials={real_user=slave}} blocked reactor thread for 92786.8us
03:08:04.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.170148 28846 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c187dbbc057548cdb1c948a1f6d94b64" instance_seqno: 1770001675792422) as {username='kudu', principal='kudu/127.27.65.2@KRBTEST.COM'} at 127.27.65.2:60003; Asking this server to re-register.
03:08:04.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.180259 28533 heartbeater.cc:461] Registering TS with master...
03:08:04.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.184335 28846 ts_manager.cc:194] Registered new tserver with Master: c187dbbc057548cdb1c948a1f6d94b64 (127.27.65.2:35089)
03:08:04.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:04.187434 28726 leader_election.cc:336] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407): Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111)
03:08:04.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.278832 28856 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4d360522e835469d8de898131ddaaafc" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "080b9f8181db40ccb2df34a96fcb4738" is_pre_election: true
03:08:04.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.280045 28856 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4d360522e835469d8de898131ddaaafc in term 1.
03:08:04.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.287597 28728 leader_election.cc:304] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 080b9f8181db40ccb2df34a96fcb4738, 4d360522e835469d8de898131ddaaafc; no voters: 040a41d80f044969aaa60adb75a339c5
03:08:04.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.288694 28920 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 1 FOLLOWER]: Leader pre-election won for term 2
03:08:04.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.289004 28920 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:08:04.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.289340 28920 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 1 FOLLOWER]: Advancing to term 2
03:08:04.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.299083 28920 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:04.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:04.305058 28726 leader_election.cc:336] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407): Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111)
03:08:04.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.306999 28920 leader_election.cc:290] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 2 election: Requested vote from peers 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407), 080b9f8181db40ccb2df34a96fcb4738 (127.27.65.60:38795)
03:08:04.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.307186 28856 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4d360522e835469d8de898131ddaaafc" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "080b9f8181db40ccb2df34a96fcb4738"
03:08:04.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.307796 28856 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 1 FOLLOWER]: Advancing to term 2
03:08:04.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.322227 28856 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4d360522e835469d8de898131ddaaafc in term 2.
03:08:04.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.349284 28728 leader_election.cc:304] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 080b9f8181db40ccb2df34a96fcb4738, 4d360522e835469d8de898131ddaaafc; no voters: 040a41d80f044969aaa60adb75a339c5
03:08:04.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.350376 28920 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Leader election won for term 2
03:08:04.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.372179 28920 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 LEADER]: Becoming Leader. State: Replica: 4d360522e835469d8de898131ddaaafc, State: Running, Role: LEADER
03:08:04.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.373451 28920 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:04.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.387756 28926 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4d360522e835469d8de898131ddaaafc. Latest consensus state: current_term: 2 leader_uuid: "4d360522e835469d8de898131ddaaafc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:04.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.391660 28926 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: This master's current role is: LEADER
03:08:04.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.400537 28928 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:08:04.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.419700 28928 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:08:04.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.424571 28928 catalog_manager.cc:1269] Loaded cluster ID: 7dd52b2ccf104be08a86cdd0bbc27a05
03:08:04.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.429376 28928 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:08:04.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.438014 28928 catalog_manager.cc:1514] Loading token signing keys...
03:08:04.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.447480 28928 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Loaded TSK: 0
03:08:04.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.452926 28928 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:08:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:04.777833 28726 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc -> Peer 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407): Couldn't send request to peer 040a41d80f044969aaa60adb75a339c5. Status: Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:08:04.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.853168 28856 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 2 FOLLOWER]: Refusing update from remote peer 4d360522e835469d8de898131ddaaafc: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
03:08:04.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.855542 28926 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [LEADER]: Connected to new peer: Peer: permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
03:08:04.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.878520 28930 sys_catalog.cc:455] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4d360522e835469d8de898131ddaaafc. Latest consensus state: current_term: 2 leader_uuid: "4d360522e835469d8de898131ddaaafc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:04.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.879503 28930 sys_catalog.cc:458] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: This master's current role is: FOLLOWER
03:08:04.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.920006 28930 sys_catalog.cc:455] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4d360522e835469d8de898131ddaaafc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:04.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.920735 28930 sys_catalog.cc:458] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: This master's current role is: FOLLOWER
03:08:04.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.921150 28920 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4d360522e835469d8de898131ddaaafc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:04.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:04.921897 28920 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: This master's current role is: LEADER
03:08:05.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:05 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001685, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:05.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:05.526262 28415 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:05.754966 28696 master_runner.cc:186] Error getting master registration for 127.27.65.62:41407: Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111),
03:08:05.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:05.755574 28696 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:06.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.214067 28534 heartbeater.cc:499] Master 127.27.65.61:42281 was elected leader, sending a full tablet report...
03:08:06.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.275664 28385 heartbeater.cc:499] Master 127.27.65.61:42281 was elected leader, sending a full tablet report...
03:08:06.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.438499 28681 heartbeater.cc:499] Master 127.27.65.61:42281 was elected leader, sending a full tablet report...
03:08:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:05.092028 28912 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 28883
03:08:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:06.453970 28912 kernel_stack_watchdog.cc:198] Thread 28883 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
03:08:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:06.455240 28883 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.766s user 0.870s sys 1.442s
03:08:06.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:06.455662 28883 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.767s user 0.870s sys 1.442s
03:08:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:03.695622 28914 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:06.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:06.456614 28915 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2767 milliseconds
03:08:06.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.458216 28883 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:06.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:06.458274 28916 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:06.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.461830 28883 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:06.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.465384 28883 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:06.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.466934 28883 hybrid_clock.cc:648] HybridClock initialized: now 1770001686466866 us; error 36 us; skew 500 ppm
03:08:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.468084 28883 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:06.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:06 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001686, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:06.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.476981 28883 init.cc:377] Logged in from keytab as kudu/127.27.65.62@KRBTEST.COM (short username kudu)
03:08:06.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.483532 28883 webserver.cc:492] Webserver started at http://127.27.65.62:36607/ using document root <none> and password file <none>
03:08:06.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.484741 28883 fs_manager.cc:362] Metadata directory not provided
03:08:06.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.485050 28883 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:06.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.525139 28883 fs_manager.cc:714] Time spent opening directory manager: real 0.034s user 0.007s sys 0.022s
03:08:06.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.551972 28948 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:06.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.553602 28883 fs_manager.cc:730] Time spent opening block manager: real 0.025s user 0.002s sys 0.020s
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.554054 28883 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "040a41d80f044969aaa60adb75a339c5"
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:42 on dist-test-slave-0wq0"
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.556973 28883 fs_report.cc:389] FS layout report
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/data
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:06.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:06.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:06.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.638266 28883 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:06.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.644723 28883 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:06.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.645251 28883 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:06.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.740247 28883 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.62:41407
03:08:06.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.740504 28999 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.62:41407 every 8 connection(s)
03:08:06.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.743136 28883 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/info.pb
03:08:06.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.747329 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 28883
03:08:06.748 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:06.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.762730 29000 sys_catalog.cc:263] Verifying existing consensus state
03:08:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.771032 29000 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Bootstrap starting.
03:08:06.820 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.65.61:42281(127.27.65.61:42281)] connecting to peer
03:08:06.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.840986 29000 log.cc:826] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Log is configured to *not* fsync() on all Append() calls
03:08:06.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.870991 29000 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:06.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.872092 29000 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Bootstrap complete.
03:08:06.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.905505 29000 raft_consensus.cc:359] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:06.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.908773 29000 raft_consensus.cc:740] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 040a41d80f044969aaa60adb75a339c5, State: Initialized, Role: FOLLOWER
03:08:06.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.909899 29000 consensus_queue.cc:260] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.924005 29005 sys_catalog.cc:455] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:06.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.924860 29005 sys_catalog.cc:458] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: This master's current role is: FOLLOWER
03:08:06.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:06.926231 29000 sys_catalog.cc:565] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: configured and running, proceeding with master startup.
03:08:06.927 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.65.60:38795(127.27.65.60:38795)] connecting to peer
03:08:06.928 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.65.62:41407(127.27.65.62:41407)] connecting to peer
03:08:07.018 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.27.65.60:38795(127.27.65.60:38795)] Successfully connected to peer
03:08:07.028 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.65.61:42281(127.27.65.61:42281)] Successfully connected to peer
03:08:07.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.092144 29021 catalog_manager.cc:1269] Loaded cluster ID: 7dd52b2ccf104be08a86cdd0bbc27a05
03:08:07.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.092444 29021 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: loading cluster ID for follower catalog manager: success
03:08:07.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.111320 29021 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: acquiring CA information for follower catalog manager: success
03:08:07.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:07 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001686, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.62@KRBTEST.COM for kudu/127.27.65.62@KRBTEST.COM
03:08:07.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.133320 29021 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:07.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.220409 28975 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [term 1 FOLLOWER]: Advancing to term 2
03:08:07.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.239657 28535 heartbeater.cc:344] Connected to a master server at 127.27.65.62:41407
03:08:07.248 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.65.62:41407(127.27.65.62:41407)] Successfully connected to peer
03:08:07.301 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
03:08:07.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.337363 29005 sys_catalog.cc:455] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4d360522e835469d8de898131ddaaafc. Latest consensus state: current_term: 2 leader_uuid: "4d360522e835469d8de898131ddaaafc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:07.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.338055 29005 sys_catalog.cc:458] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: This master's current role is: FOLLOWER
03:08:07.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.329453 29036 sys_catalog.cc:455] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "4d360522e835469d8de898131ddaaafc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.338728 29036 sys_catalog.cc:458] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5 [sys.catalog]: This master's current role is: FOLLOWER
03:08:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.338795 28965 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c187dbbc057548cdb1c948a1f6d94b64" instance_seqno: 1770001675792422) as {username='kudu', principal='kudu/127.27.65.2@KRBTEST.COM'} at 127.27.65.2:41987; Asking this server to re-register.
03:08:07.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.341015 28535 heartbeater.cc:461] Registering TS with master...
03:08:07.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.345458 28965 ts_manager.cc:194] Registered new tserver with Master: c187dbbc057548cdb1c948a1f6d94b64 (127.27.65.2:35089)
03:08:07.350 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.27.65.61
03:08:07.350 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.27.65.60
03:08:07.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.357882 28389 heartbeater.cc:344] Connected to a master server at 127.27.65.62:41407
03:08:07.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.360742 28965 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c32ac7effcab4a3b92bf2b6f555a0c1e" instance_seqno: 1770001672936085) as {username='kudu', principal='kudu/127.27.65.1@KRBTEST.COM'} at 127.27.65.1:55637; Asking this server to re-register.
03:08:07.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.362397 28389 heartbeater.cc:461] Registering TS with master...
03:08:07.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.365558 28965 ts_manager.cc:194] Registered new tserver with Master: c32ac7effcab4a3b92bf2b6f555a0c1e (127.27.65.1:35391)
03:08:07.522 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.27.65.62
03:08:07.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.560470 28684 heartbeater.cc:344] Connected to a master server at 127.27.65.62:41407
03:08:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.561383 29025 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "4d360522e835469d8de898131ddaaafc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:07.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.562059 29025 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: This master's current role is: LEADER
03:08:07.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.563500 28965 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "61749f6f946d469197f1bd9e080f9108" instance_seqno: 1770001678109585) as {username='kudu', principal='kudu/127.27.65.3@KRBTEST.COM'} at 127.27.65.3:38081; Asking this server to re-register.
03:08:07.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.564906 28684 heartbeater.cc:461] Registering TS with master...
03:08:07.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.569558 28965 ts_manager.cc:194] Registered new tserver with Master: 61749f6f946d469197f1bd9e080f9108 (127.27.65.3:37113)
03:08:07.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:07 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001678, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.65.61@KRBTEST.COM
03:08:07.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:07 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001678, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:08:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:07 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001678, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.65.62@KRBTEST.COM
03:08:07.808 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x856c853f, L:/127.0.0.1:34488 - R:/127.27.65.61:42281] using SASL/GSSAPI
03:08:07.810 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x3411a465, L:/127.0.0.1:47448 - R:/127.27.65.60:38795] using SASL/GSSAPI
03:08:07.814 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x11aa90b8, L:/127.0.0.1:55826 - R:/127.27.65.62:41407] using SASL/GSSAPI
03:08:07.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:07 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001687, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:07.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:07.846072 28562 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:07.879 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:08:07.880 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1125), Traces: [0ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.65.61:42281, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.65.60:38795, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.65.62:41407, [1072ms] Sub RPC ConnectToMaster: received response from server master-127.27.65.62:41407: OK, [1083ms] Sub RPC ConnectToMaster: received response from server master-127.27.65.61:42281: OK, [1086ms] Sub RPC ConnectToMaster: received response from server master-127.27.65.60:38795: OK, deferred=null) after lookup
03:08:08.903 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:08:08.903 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:08:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:08 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001688, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:08:08.937 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:08.938 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc
03:08:08.939 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:08:08.939 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.65.61:42281
03:08:08.939 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:08:08.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:08.940443 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 28696
03:08:08.970 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:08.971 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.65.60:38795
03:08:08.972 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:08:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:08.972743 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 28779
03:08:08.983 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.65.61:42281(127.27.65.61:42281)] handling channelInactive
03:08:08.984 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.65.61:42281(127.27.65.61:42281)] cleaning up while in state READY due to: connection closed
03:08:09.016 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.27.65.60:38795(127.27.65.60:38795)] handling channelInactive
03:08:09.016 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:09.016 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.65.62:41407
03:08:09.016 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.65.60:38795(127.27.65.60:38795)] cleaning up while in state READY due to: connection closed
03:08:09.016 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:08:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:09.017251 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 28883
03:08:09.048 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.65.62:41407(127.27.65.62:41407)] handling channelInactive
03:08:09.048 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.65.62:41407(127.27.65.62:41407)] cleaning up while in state READY due to: connection closed
03:08:09.049 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:09.049 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.65.61:42281
03:08:09.049 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:09.051317 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/logs
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/info.pb
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:09.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.61:42281
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.61
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36875
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.61
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:09.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:09.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.418351 29046 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.418998 29046 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.419703 29046 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:09.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.452929 29046 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:09.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.453229 29046 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:09.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.453675 29046 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:09.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.453994 29046 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:09.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.454321 29046 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:09.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.454717 29046 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:09.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.454986 29046 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42881/jwks.json
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:09.498610 29046 master_runner.cc:386] Master server non-default flags:
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.61
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.61:42281
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:09.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/info.pb
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.61
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36875
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/logs
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:09.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:09.500311 29046 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:09.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:09.501573 29046 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:09.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.523145 29057 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:10.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:10.114831 27610 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac700 after lost signal to thread 27480
03:08:10.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:10.117275 27610 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac520 after lost signal to thread 27613
03:08:10.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:10.973886 27893 debug-util.cc:398] Leaking SignalData structure 0x7b08000b95c0 after lost signal to thread 27766
03:08:10.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:10.975567 27893 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad660 after lost signal to thread 27896
03:08:11.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.523358 29056 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:11.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:09.541154 29059 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:11.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.042300 29046 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:11.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:11.042969 29058 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1517 milliseconds
03:08:11.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.044672 29046 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:11.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.048213 29046 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:11.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.049715 29046 hybrid_clock.cc:648] HybridClock initialized: now 1770001691049668 us; error 55 us; skew 500 ppm
03:08:11.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.051009 29046 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:11.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:11 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001691, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:11.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.059654 29046 init.cc:377] Logged in from keytab as kudu/127.27.65.61@KRBTEST.COM (short username kudu)
03:08:11.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.067273 29046 webserver.cc:492] Webserver started at http://127.27.65.61:36875/ using document root <none> and password file <none>
03:08:11.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.068646 29046 fs_manager.cc:362] Metadata directory not provided
03:08:11.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.068992 29046 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:11.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.082618 29046 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.006s sys 0.003s
03:08:11.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.090947 29071 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:11.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.092522 29046 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.004s sys 0.001s
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.092996 29046 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4d360522e835469d8de898131ddaaafc"
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:45 on dist-test-slave-0wq0"
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.096334 29046 fs_report.cc:389] FS layout report
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/wal
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/data
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:11.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:11.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.193507 29046 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:11.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.211887 29046 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:11.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.212491 29046 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:11.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.386981 29046 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.61:42281
03:08:11.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.387115 29126 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.61:42281 every 8 connection(s)
03:08:11.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.391782 29046 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-1/data/info.pb
03:08:11.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.398221 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 29046
03:08:11.399 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:11.399 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.65.60:38795
03:08:11.399 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:08:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.401393 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:08:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:08:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:08:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data
03:08:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:11.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/logs
03:08:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/info.pb
03:08:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:11.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.60:38795
03:08:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.60
03:08:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41997
03:08:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:08:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.60
03:08:11.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:11.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:11.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.425365 29127 sys_catalog.cc:263] Verifying existing consensus state
03:08:11.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.435827 29127 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Bootstrap starting.
03:08:11.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:11 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001687, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.3@KRBTEST.COM for kudu/127.27.65.61@KRBTEST.COM
03:08:11.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.602288 28681 heartbeater.cc:344] Connected to a master server at 127.27.65.61:42281
03:08:11.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.602470 29127 log.cc:826] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Log is configured to *not* fsync() on all Append() calls
03:08:11.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.638464 29127 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:11.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.639626 29127 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Bootstrap complete.
03:08:11.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.674700 29127 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:11.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.678830 29127 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4d360522e835469d8de898131ddaaafc, State: Initialized, Role: FOLLOWER
03:08:11.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.680922 29127 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:11.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.689980 29134 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:11.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.693864 29127 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: configured and running, proceeding with master startup.
03:08:11.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.697749 29134 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: This master's current role is: FOLLOWER
03:08:11.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.771711 29145 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:08:11.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.790462 29046 master_runner.cc:186] Error getting master registration for 127.27.65.62:41407: Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111),
03:08:11.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:11.824082 29046 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:11.894840 29129 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:11.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:11.895604 29129 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:11.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:11.896387 29129 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:11.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:11.929641 29129 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:11.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:11.929922 29129 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:11.930374 29129 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:11.930727 29129 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:11.931084 29129 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:11.931422 29129 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:11.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:11.931687 29129 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42881/jwks.json
03:08:12.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.003178 29129 master_runner.cc:386] Master server non-default flags:
03:08:12.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.60
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.60:38795
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/info.pb
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.60
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41997
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/logs
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:12.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:08:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:08:12.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.005136 29129 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:12.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.006613 29129 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:12.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:12.021472 29152 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:12.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:12 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001685, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.2@KRBTEST.COM for kudu/127.27.65.61@KRBTEST.COM
03:08:12.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.324187 28534 heartbeater.cc:344] Connected to a master server at 127.27.65.61:42281
03:08:12.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.330667 29090 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c187dbbc057548cdb1c948a1f6d94b64" instance_seqno: 1770001675792422) as {username='kudu', principal='kudu/127.27.65.2@KRBTEST.COM'} at 127.27.65.2:47497; Asking this server to re-register.
03:08:12.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.332808 28534 heartbeater.cc:461] Registering TS with master...
03:08:12.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.336792 29089 ts_manager.cc:194] Registered new tserver with Master: c187dbbc057548cdb1c948a1f6d94b64 (127.27.65.2:35089)
03:08:12.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:12.375560 29157 negotiation.cc:337] Failed RPC negotiation. Trace:
03:08:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:12.344832 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.27.65.1:57683 (local address 127.27.65.61:42281)
03:08:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:12.345834 (+ 1002us) server_negotiation.cc:207] Beginning negotiation
03:08:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:12.345867 (+ 33us) server_negotiation.cc:400] Waiting for connection header
03:08:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:12.366185 (+ 20318us) server_negotiation.cc:408] Connection header received
03:08:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:12.369776 (+ 3591us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
03:08:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:12.369830 (+ 54us) server_negotiation.cc:462] Received NEGOTIATE request from client
03:08:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:12.370223 (+ 393us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
03:08:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:12.370425 (+ 202us) server_negotiation.cc:227] Negotiated authn=SASL
03:08:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:12.373393 (+ 2968us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.27.65.1:57683: BlockingRecv error: recv got EOF from 127.27.65.1:57683 (error 108)
03:08:12.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":146}
03:08:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:12 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001692, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:12.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.567528 28263 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:12.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.610785 29090 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "61749f6f946d469197f1bd9e080f9108" instance_seqno: 1770001678109585) as {username='kudu', principal='kudu/127.27.65.3@KRBTEST.COM'} at 127.27.65.3:34477; Asking this server to re-register.
03:08:12.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.614231 28681 heartbeater.cc:461] Registering TS with master...
03:08:12.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.618664 29090 ts_manager.cc:194] Registered new tserver with Master: 61749f6f946d469197f1bd9e080f9108 (127.27.65.3:37113)
03:08:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.815840 29145 catalog_manager.cc:1269] Loaded cluster ID: 7dd52b2ccf104be08a86cdd0bbc27a05
03:08:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.816256 29145 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: loading cluster ID for follower catalog manager: success
03:08:12.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.834478 29145 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: acquiring CA information for follower catalog manager: success
03:08:12.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:12.862607 29145 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:13.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.312187 29165 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:13.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.313134 29165 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:13.325873 29072 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111)
03:08:13.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.329208 29165 leader_election.cc:290] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407), 080b9f8181db40ccb2df34a96fcb4738 (127.27.65.60:38795)
03:08:13.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:13.334542 29072 leader_election.cc:336] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407): Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111)
03:08:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:13.351821 29074 leader_election.cc:336] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 080b9f8181db40ccb2df34a96fcb4738 (127.27.65.60:38795): Network error: Client connection negotiation failed: client connection to 127.27.65.60:38795: connect: Connection refused (error 111)
03:08:13.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.353137 29074 leader_election.cc:304] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4d360522e835469d8de898131ddaaafc; no voters: 040a41d80f044969aaa60adb75a339c5, 080b9f8181db40ccb2df34a96fcb4738
03:08:13.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.354866 29165 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
03:08:13.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.458395 28385 heartbeater.cc:344] Connected to a master server at 127.27.65.61:42281
03:08:13.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.461259 29090 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c32ac7effcab4a3b92bf2b6f555a0c1e" instance_seqno: 1770001672936085) as {username='kudu', principal='kudu/127.27.65.1@KRBTEST.COM'} at 127.27.65.1:50553; Asking this server to re-register.
03:08:13.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.462816 28385 heartbeater.cc:461] Registering TS with master...
03:08:13.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.465904 29090 ts_manager.cc:194] Registered new tserver with Master: c32ac7effcab4a3b92bf2b6f555a0c1e (127.27.65.1:35391)
03:08:13.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:13.422894 29151 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 29129
03:08:13.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:12.023236 29153 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:13.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:13.806607 29129 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.786s user 0.570s sys 0.884s
03:08:13.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:13.807719 29129 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.787s user 0.570s sys 0.884s
03:08:13.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.815449 29129 server_base.cc:1060] running on GCE node
03:08:13.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:13.817222 29176 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:13.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.818795 29129 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.822142 29129 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:13.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.823772 29129 hybrid_clock.cc:648] HybridClock initialized: now 1770001693823733 us; error 42 us; skew 500 ppm
03:08:13.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.824900 29129 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:13.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:13 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001693, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:13.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.838065 29129 init.cc:377] Logged in from keytab as kudu/127.27.65.60@KRBTEST.COM (short username kudu)
03:08:13.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.855480 29129 webserver.cc:492] Webserver started at http://127.27.65.60:41997/ using document root <none> and password file <none>
03:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.857218 29129 fs_manager.cc:362] Metadata directory not provided
03:08:13.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.857518 29129 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:13.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.869973 29129 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.003s sys 0.003s
03:08:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.876575 29182 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:13.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.878016 29129 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
03:08:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.878572 29129 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:08:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "080b9f8181db40ccb2df34a96fcb4738"
03:08:13.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:47 on dist-test-slave-0wq0"
03:08:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.882092 29129 fs_report.cc:389] FS layout report
03:08:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:08:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/wal
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/data
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:13.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.975394 29129 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:13.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.984406 29129 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:13.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:13.985229 29129 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:14.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.091117 29129 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.60:38795
03:08:14.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.091228 29233 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.60:38795 every 8 connection(s)
03:08:14.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.095543 29129 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-2/data/info.pb
03:08:14.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.104597 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 29129
03:08:14.105 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:14.105 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.65.62:41407
03:08:14.106 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.107767 27908 external_mini_cluster.cc:1366] Running /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/logs
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/info.pb
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:08:14.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.62:41407
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.62
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36607
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.62
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:08:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:08:14.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.153360 29234 sys_catalog.cc:263] Verifying existing consensus state
03:08:14.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.162798 29234 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Bootstrap starting.
03:08:14.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.258312 29234 log.cc:826] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Log is configured to *not* fsync() on all Append() calls
03:08:14.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.298146 29234 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:14.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.299173 29234 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: Bootstrap complete.
03:08:14.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.358850 29234 raft_consensus.cc:359] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:14.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.362391 29234 raft_consensus.cc:740] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 080b9f8181db40ccb2df34a96fcb4738, State: Initialized, Role: FOLLOWER
03:08:14.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.363411 29234 consensus_queue.cc:260] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:14.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:14 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001685, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.2@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:08:14.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.382300 29240 sys_catalog.cc:455] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:14.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.387199 29240 sys_catalog.cc:458] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: This master's current role is: FOLLOWER
03:08:14.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.384974 29234 sys_catalog.cc:565] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: configured and running, proceeding with master startup.
03:08:14.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.416674 28533 heartbeater.cc:344] Connected to a master server at 127.27.65.60:38795
03:08:14.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:14 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001692, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.1@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:08:14.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.514461 28384 heartbeater.cc:344] Connected to a master server at 127.27.65.60:38795
03:08:14.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.521904 29254 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:08:14.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.569319 29129 master_runner.cc:186] Error getting master registration for 127.27.65.62:41407: Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111),
03:08:14.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.598994 29129 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:14.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:14 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001687, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.3@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:08:14.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.623802 28680 heartbeater.cc:344] Connected to a master server at 127.27.65.60:38795
03:08:14.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.629860 29199 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "61749f6f946d469197f1bd9e080f9108" instance_seqno: 1770001678109585) as {username='kudu', principal='kudu/127.27.65.3@KRBTEST.COM'} at 127.27.65.3:48141; Asking this server to re-register.
03:08:14.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.631409 28680 heartbeater.cc:461] Registering TS with master...
03:08:14.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.634897 29199 ts_manager.cc:194] Registered new tserver with Master: 61749f6f946d469197f1bd9e080f9108 (127.27.65.3:37113)
03:08:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.712170 29235 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:08:14.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.712713 29235 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:08:14.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.713336 29235 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:08:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.747035 29235 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:08:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.747272 29235 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:08:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.747531 29235 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:08:14.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.747740 29235 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:08:14.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.748042 29235 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:08:14.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.748250 29235 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:08:14.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.748411 29235 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42881/jwks.json
03:08:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.788540 29235 master_runner.cc:386] Master server non-default flags:
03:08:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:08:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.65.20:36425
03:08:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:08:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:08:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data
03:08:14.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.65.62:41407,127.27.65.61:42281,127.27.65.60:38795
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/krb5kdc/kudu.keytab
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.65.62
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.65.62:41407
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42881/jwks.json
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/info.pb
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.65.62
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36607
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/logs
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:08:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3722ed49c19ce54c7feae2fa944e32b1d4ed8d2d
03:08:14.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:08:14.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 02 Feb 2026 02:43:23 UTC on 5fd53c4cbb9d
03:08:14.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 10305
03:08:14.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:08:14.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.789954 29235 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:08:14.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.790964 29235 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:08:14.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.802268 29263 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:14.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.904239 29269 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:08:14.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.905138 29269 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:14.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.926770 29269 leader_election.cc:290] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407), 080b9f8181db40ccb2df34a96fcb4738 (127.27.65.60:38795)
03:08:14.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.958436 29072 leader_election.cc:336] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407): Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111)
03:08:14.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:14 dist-test-slave-0wq0 krb5kdc[27921](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001691, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.61@KRBTEST.COM for kudu/127.27.65.60@KRBTEST.COM
03:08:14.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.991364 29209 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4d360522e835469d8de898131ddaaafc" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "080b9f8181db40ccb2df34a96fcb4738" is_pre_election: true
03:08:14.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.992508 29209 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4d360522e835469d8de898131ddaaafc in term 2.
03:08:14.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.994208 29074 leader_election.cc:304] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 080b9f8181db40ccb2df34a96fcb4738, 4d360522e835469d8de898131ddaaafc; no voters: 040a41d80f044969aaa60adb75a339c5
03:08:14.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.996104 29269 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Leader pre-election won for term 3
03:08:14.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.996449 29269 raft_consensus.cc:493] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:08:14.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:14.996765 29269 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 2 FOLLOWER]: Advancing to term 3
03:08:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.006273 29269 raft_consensus.cc:515] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.016101 29209 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4d360522e835469d8de898131ddaaafc" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "080b9f8181db40ccb2df34a96fcb4738"
03:08:15.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.016680 29209 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 2 FOLLOWER]: Advancing to term 3
03:08:15.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:15.025609 29072 leader_election.cc:336] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407): Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111)
03:08:15.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.030231 29209 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4d360522e835469d8de898131ddaaafc in term 3.
03:08:15.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.035012 29269 leader_election.cc:290] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 3 election: Requested vote from peers 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407), 080b9f8181db40ccb2df34a96fcb4738 (127.27.65.60:38795)
03:08:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.036433 29074 leader_election.cc:304] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 080b9f8181db40ccb2df34a96fcb4738, 4d360522e835469d8de898131ddaaafc; no voters: 040a41d80f044969aaa60adb75a339c5
03:08:15.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.037683 29269 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 3 FOLLOWER]: Leader election won for term 3
03:08:15.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.043263 29269 raft_consensus.cc:697] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [term 3 LEADER]: Becoming Leader. State: Replica: 4d360522e835469d8de898131ddaaafc, State: Running, Role: LEADER
03:08:15.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.044392 29269 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } }
03:08:15.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.067827 29273 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4d360522e835469d8de898131ddaaafc. Latest consensus state: current_term: 3 leader_uuid: "4d360522e835469d8de898131ddaaafc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:15.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.070603 29273 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: This master's current role is: LEADER
03:08:15.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.083778 29275 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:08:15.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.126626 29275 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:08:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.153426 29275 catalog_manager.cc:1269] Loaded cluster ID: 7dd52b2ccf104be08a86cdd0bbc27a05
03:08:15.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.153806 29275 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:08:15.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.195417 29275 catalog_manager.cc:1514] Loading token signing keys...
03:08:15.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.205888 29275 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc: Loaded TSK: 0
03:08:15.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.218259 29275 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:08:15.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.425156 29199 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c187dbbc057548cdb1c948a1f6d94b64" instance_seqno: 1770001675792422) as {username='kudu', principal='kudu/127.27.65.2@KRBTEST.COM'} at 127.27.65.2:51357; Asking this server to re-register.
03:08:15.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.427662 28533 heartbeater.cc:461] Registering TS with master...
03:08:15.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.432749 29199 ts_manager.cc:194] Registered new tserver with Master: c187dbbc057548cdb1c948a1f6d94b64 (127.27.65.2:35089)
03:08:15.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.507037 29209 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [term 3 FOLLOWER]: Refusing update from remote peer 4d360522e835469d8de898131ddaaafc: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
03:08:15.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.508992 29273 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [LEADER]: Connected to new peer: Peer: permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
03:08:15.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:15 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001695, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:15.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.530552 28415 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:15.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.549104 29199 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c32ac7effcab4a3b92bf2b6f555a0c1e" instance_seqno: 1770001672936085) as {username='kudu', principal='kudu/127.27.65.1@KRBTEST.COM'} at 127.27.65.1:37253; Asking this server to re-register.
03:08:15.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.550527 28384 heartbeater.cc:461] Registering TS with master...
03:08:15.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.553063 29279 sys_catalog.cc:455] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4d360522e835469d8de898131ddaaafc. Latest consensus state: current_term: 3 leader_uuid: "4d360522e835469d8de898131ddaaafc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:15.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.553838 29279 sys_catalog.cc:458] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: This master's current role is: FOLLOWER
03:08:15.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.553967 29199 ts_manager.cc:194] Registered new tserver with Master: c32ac7effcab4a3b92bf2b6f555a0c1e (127.27.65.1:35391)
03:08:15.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.594429 29273 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "4d360522e835469d8de898131ddaaafc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:15.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.595151 29273 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc [sys.catalog]: This master's current role is: LEADER
03:08:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.600235 29254 catalog_manager.cc:1269] Loaded cluster ID: 7dd52b2ccf104be08a86cdd0bbc27a05
03:08:15.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:15.620348 29072 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc -> Peer 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407): Couldn't send request to peer 040a41d80f044969aaa60adb75a339c5. Status: Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:08:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.621297 29254 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: loading cluster ID for follower catalog manager: success
03:08:15.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.616308 29279 sys_catalog.cc:455] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "4d360522e835469d8de898131ddaaafc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "040a41d80f044969aaa60adb75a339c5" member_type: VOTER last_known_addr { host: "127.27.65.62" port: 41407 } } peers { permanent_uuid: "4d360522e835469d8de898131ddaaafc" member_type: VOTER last_known_addr { host: "127.27.65.61" port: 42281 } } peers { permanent_uuid: "080b9f8181db40ccb2df34a96fcb4738" member_type: VOTER last_known_addr { host: "127.27.65.60" port: 38795 } } }
03:08:15.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.624482 29279 sys_catalog.cc:458] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738 [sys.catalog]: This master's current role is: FOLLOWER
03:08:15.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.644678 29254 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: acquiring CA information for follower catalog manager: success
03:08:15.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:15.667249 29254 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 080b9f8181db40ccb2df34a96fcb4738: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:08:16.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:16.370030 28534 heartbeater.cc:499] Master 127.27.65.61:42281 was elected leader, sending a full tablet report...
03:08:16.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:16.475703 28385 heartbeater.cc:499] Master 127.27.65.61:42281 was elected leader, sending a full tablet report...
03:08:16.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:16.635946 28681 heartbeater.cc:499] Master 127.27.65.61:42281 was elected leader, sending a full tablet report...
03:08:16.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:16.827773 29046 master_runner.cc:186] Error getting master registration for 127.27.65.62:41407: Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111),
03:08:16.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:16.828348 29046 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:08:17.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:17 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001697, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:17.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:17.850529 28562 init.cc:295] Successfully reacquired a new kerberos TGT
03:08:17.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:16.204548 29262 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 29235
03:08:17.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:17.971014 29262 kernel_stack_watchdog.cc:198] Thread 29235 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
03:08:17.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:08:17.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:08:17.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:08:17.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:08:17.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:14.803619 29264 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:17.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:17.972270 29235 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.169s user 1.001s sys 1.620s
03:08:17.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:17.972617 29235 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.169s user 1.001s sys 1.620s
03:08:17.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:17.974103 29266 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:08:17.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:17.976711 29235 server_base.cc:1055] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:08:17.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:17.976711 29265 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 3172 milliseconds
03:08:17.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:17.978245 29235 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:08:17.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:17.982019 29235 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:08:17.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:17.983518 29235 hybrid_clock.cc:648] HybridClock initialized: now 1770001697983415 us; error 114 us; skew 500 ppm
03:08:17.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:17.985430 29235 server_base.cc:860] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:08:17.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Feb 02 03:08:17 dist-test-slave-0wq0 krb5kdc[27921](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1770001697, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.65.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:08:17.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:17.993894 29235 init.cc:377] Logged in from keytab as kudu/127.27.65.62@KRBTEST.COM (short username kudu)
03:08:18.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.000770 29235 webserver.cc:492] Webserver started at http://127.27.65.62:36607/ using document root <none> and password file <none>
03:08:18.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.001677 29235 fs_manager.cc:362] Metadata directory not provided
03:08:18.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.001874 29235 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:08:18.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.016810 29235 fs_manager.cc:714] Time spent opening directory manager: real 0.012s user 0.005s sys 0.002s
03:08:18.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.027204 29306 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:08:18.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.030433 29235 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.002s sys 0.001s
03:08:18.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.031042 29235 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data,/tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:08:18.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "040a41d80f044969aaa60adb75a339c5"
03:08:18.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-02-02 03:07:42 on dist-test-slave-0wq0"
03:08:18.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.034415 29235 fs_report.cc:389] FS layout report
03:08:18.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/wal
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/data
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:08:18.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:08:18.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.117713 29235 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:08:18.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.132540 29235 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:08:18.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.133075 29235 kserver.cc:163] Server-wide thread pool size limit: 3276
03:08:18.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:18.190347 29072 consensus_peers.cc:597] T 00000000000000000000000000000000 P 4d360522e835469d8de898131ddaaafc -> Peer 040a41d80f044969aaa60adb75a339c5 (127.27.65.62:41407): Couldn't send request to peer 040a41d80f044969aaa60adb75a339c5. Status: Network error: Client connection negotiation failed: client connection to 127.27.65.62:41407: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:08:18.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.229828 29235 rpc_server.cc:307] RPC server started. Bound to: 127.27.65.62:41407
03:08:18.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.229938 29358 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.65.62:41407 every 8 connection(s)
03:08:18.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.233418 29235 server_base.cc:1192] Dumped server information to /tmp/dist-test-tasktpBrrC/test-tmp/mini-kudu-cluster4957187322515487766/master-0/data/info.pb
03:08:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.236680 27908 external_mini_cluster.cc:1428] Started /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu as pid 29235
03:08:18.237 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:08:18.238 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.65.61:42281(127.27.65.61:42281)] connecting to peer
03:08:18.242 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.27.65.61:42281(127.27.65.61:42281)] Successfully connected to peer
03:08:18.245 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.65.61:42281(127.27.65.61:42281)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
03:08:18.246 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.27.65.61:42281(127.27.65.61:42281)] handling channelInactive
03:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:18.248801 29361 negotiation.cc:337] Failed RPC negotiation. Trace:
03:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:18.242022 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:35222 (local address 127.27.65.61:42281)
03:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:18.243432 (+ 1410us) server_negotiation.cc:207] Beginning negotiation
03:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:18.243546 (+ 114us) server_negotiation.cc:400] Waiting for connection header
03:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:18.243707 (+ 161us) server_negotiation.cc:408] Connection header received
03:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:18.243846 (+ 139us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
03:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:18.243873 (+ 27us) server_negotiation.cc:462] Received NEGOTIATE request from client
03:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:18.244084 (+ 211us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
03:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:18.245399 (+ 1315us) server_negotiation.cc:227] Negotiated authn=SASL
03:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0202 03:08:18.247645 (+ 2246us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:35222: BlockingRecv error: recv got EOF from 127.0.0.1:35222 (error 108)
03:08:18.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":768,"thread_start_us":3035,"threads_started":1}
03:08:18.249 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.65.62:41407(127.27.65.62:41407)] disconnecting while in state TERMINATED
03:08:18.251 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.65.60:38795(127.27.65.60:38795)] disconnecting while in state TERMINATED
03:08:18.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.251513 29359 sys_catalog.cc:263] Verifying existing consensus state
03:08:18.252 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.65.61:42281(127.27.65.61:42281)] disconnecting while in state TERMINATED
03:08:18.255 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:08:18.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.258139 29359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Bootstrap starting.
03:08:18.262 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:08:18.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.268105 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 28248
03:08:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.303818 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 28399
03:08:18.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.321431 29359 log.cc:826] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Log is configured to *not* fsync() on all Append() calls
03:08:18.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.336090 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 28547
03:08:18.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.345269 29359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:08:18.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.346307 29359 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 040a41d80f044969aaa60adb75a339c5: Bootstrap complete.
03:08:18.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.374130 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 29235
03:08:18.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.400360 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 29046
03:08:18.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:18.442584 27908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 29129
03:08:19.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:19.055877 27332 debug-util.cc:398] Leaking SignalData structure 0x7b08000b07c0 after lost signal to thread 27267
03:08:19.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260202 03:08:19.056435 27332 debug-util.cc:398] Leaking SignalData structure 0x7b08000b0d60 after lost signal to thread 27335
03:08:19.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T03:08:19Z chronyd exiting
03:08:20.756 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:08:20.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:20.758549 27181 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 27459
03:08:20.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:20.787900 27181 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 27618
03:08:20.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:20.825693 27181 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 27765
03:08:20.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:20.855293 27181 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 27193
03:08:20.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:20.886744 27181 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 27263
03:08:20.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260202 03:08:20.921263 27181 external_mini_cluster.cc:1658] Killing /tmp/dist-test-tasktpBrrC/build/tsan/bin/kudu with pid 27338
03:08:22.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-02-02T03:08:22Z chronyd exiting
03:08:23.120 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more