15:10:14.966 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
15:10:14.966 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskrL_IJG/test-tmp
15:10:14.967 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:10:14.967 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskrL_IJG/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:10:14.970 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
15:10:15.294 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:10:15.295 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:10:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:15.297897 26157 mini_oidc.cc:150] Starting JWKS server
15:10:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:15.301236 26157 webserver.cc:492] Webserver started at http://127.0.0.1:44893/ using document root <none> and password file <none>
15:10:15.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:15.301653 26157 mini_oidc.cc:186] Starting OIDC Discovery server
15:10:15.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:15.303326 26157 webserver.cc:492] Webserver started at http://127.0.0.1:36119/ using document root <none> and password file <none>
15:10:15.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:10:15Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:10:15.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:10:15Z Disabled control of system clock
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:15.364722 26157 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/wal
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/data
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/logs
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/data/info.pb
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.126:46321
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.126
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:15.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44893/jwks.json with env {}
15:10:15.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.660739 26169 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:15.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.661257 26169 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:15.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.661759 26169 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:15.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.693730 26169 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:15.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.694039 26169 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:15.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.694264 26169 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:15.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.694495 26169 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:15.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.694697 26169 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:15.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.694846 26169 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44893/jwks.json
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:15.731251 26169 master_runner.cc:386] Master server non-default flags:
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/data
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/wal
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.126:46321
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:15.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44893/jwks.json
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/data/info.pb
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.126
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/logs
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:15.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:15.732633 26169 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:15.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:15.734346 26169 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:15.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.744560 26175 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:16.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.749800 26178 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:16.931275 26177 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
15:10:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:15.746492 26176 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:16.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.931362 26169 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:16.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.935717 26169 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:16.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.938900 26169 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:16.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.940367 26169 hybrid_clock.cc:648] HybridClock initialized: now 1765638616940324 us; error 62 us; skew 500 ppm
15:10:16.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.941212 26169 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:16.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.948280 26169 webserver.cc:492] Webserver started at http://127.25.139.126:34901/ using document root <none> and password file <none>
15:10:16.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.949201 26169 fs_manager.cc:362] Metadata directory not provided
15:10:16.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.949397 26169 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:16.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.949808 26169 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:10:16.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.954257 26169 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/data/instance:
15:10:16.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec"
15:10:16.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:16 on dist-test-slave-xht5"
15:10:16.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.955355 26169 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/wal/instance:
15:10:16.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec"
15:10:16.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:16 on dist-test-slave-xht5"
15:10:16.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.962502 26169 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
15:10:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.968305 26185 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:16.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.969461 26169 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.969761 26169 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/wal
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec"
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:16 on dist-test-slave-xht5"
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:16.970044 26169 fs_report.cc:389] FS layout report
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/wal
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/wal
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/data/data
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:16.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:17.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.021905 26169 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:17.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.023394 26169 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:17.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.023797 26169 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.093571 26169 rpc_server.cc:307] RPC server started. Bound to: 127.25.139.126:46321
15:10:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.093638 26236 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.139.126:46321 every 8 connection(s)
15:10:17.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.096397 26169 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/data/info.pb
15:10:17.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.102173 26237 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:17.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.106868 26157 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 26169
15:10:17.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.107316 26157 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-0/wal/instance
15:10:17.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.110669 26157 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/wal
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/data
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/logs
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/data/info.pb
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.125:44983
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.125
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:17.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:17.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:17.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:17.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:17.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44893/jwks.json with env {}
15:10:17.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.112797 26237 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } has no permanent_uuid. Determining permanent_uuid...
15:10:17.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.152347 26237 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } has no permanent_uuid. Determining permanent_uuid...
15:10:17.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.154899 26188 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.139.125:44983: connect: Connection refused (error 111)
15:10:17.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.159089 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.125:44983: Network error: Client connection negotiation failed: client connection to 127.25.139.125:44983: connect: Connection refused (error 111)
15:10:17.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.208686 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } attempt: 1
15:10:17.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.212612 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.125:44983: Network error: Client connection negotiation failed: client connection to 127.25.139.125:44983: connect: Connection refused (error 111)
15:10:17.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.281104 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } attempt: 2
15:10:17.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.284902 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.125:44983: Network error: Client connection negotiation failed: client connection to 127.25.139.125:44983: connect: Connection refused (error 111)
15:10:17.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.376376 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } attempt: 3
15:10:17.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.380303 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.125:44983: Network error: Client connection negotiation failed: client connection to 127.25.139.125:44983: connect: Connection refused (error 111)
15:10:17.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.421505 26239 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:17.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.422044 26239 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:17.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.422616 26239 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.454720 26239 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.455231 26239 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:17.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.455610 26239 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.455852 26239 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.456048 26239 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:17.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.456200 26239 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44893/jwks.json
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.492473 26239 master_runner.cc:386] Master server non-default flags:
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/data
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/wal
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.125:44983
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44893/jwks.json
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/data/info.pb
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.125
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/logs
15:10:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:17.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.494134 26239 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.495853 26239 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:17.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.507896 26248 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:17.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.523795 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } attempt: 4
15:10:17.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.531270 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.125:44983: Network error: Client connection negotiation failed: client connection to 127.25.139.125:44983: connect: Connection refused (error 111)
15:10:17.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:17.830839 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } attempt: 5
15:10:17.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.838526 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.125:44983: Network error: Client connection negotiation failed: client connection to 127.25.139.125:44983: connect: Connection refused (error 111)
15:10:18.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:18.389746 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } attempt: 6
15:10:18.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:18.418529 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.125:44983: Network error: Client connection negotiation failed: client connection to 127.25.139.125:44983: connect: Connection refused (error 111)
15:10:19.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:17.508500 26249 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:19.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:18.911700 26247 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec80 after lost signal to thread 26239
15:10:19.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.267652 26239 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.760s user 0.687s sys 1.066s
15:10:19.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.269275 26239 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.762s user 0.687s sys 1.066s
15:10:19.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.269800 26250 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1760 milliseconds
15:10:19.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.269882 26239 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:19.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.269946 26251 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:19.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.273670 26239 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:19.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.276108 26239 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:19.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.277472 26239 hybrid_clock.cc:648] HybridClock initialized: now 1765638619277441 us; error 55 us; skew 500 ppm
15:10:19.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.278275 26239 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:19.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.284575 26239 webserver.cc:492] Webserver started at http://127.25.139.125:37941/ using document root <none> and password file <none>
15:10:19.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.285459 26239 fs_manager.cc:362] Metadata directory not provided
15:10:19.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.285643 26239 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:19.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.286029 26239 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:10:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.290354 26239 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/data/instance:
15:10:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f24086a02154b8ba60587ba1a8d2d42"
15:10:19.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:19 on dist-test-slave-xht5"
15:10:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.291453 26239 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/wal/instance:
15:10:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f24086a02154b8ba60587ba1a8d2d42"
15:10:19.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:19 on dist-test-slave-xht5"
15:10:19.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.298465 26239 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.002s
15:10:19.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.304251 26260 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:19.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.305343 26239 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.305688 26239 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/wal
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9f24086a02154b8ba60587ba1a8d2d42"
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:19 on dist-test-slave-xht5"
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.306006 26239 fs_report.cc:389] FS layout report
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/wal
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/wal
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/data/data
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:19.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.369586 26239 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.371060 26239 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:19.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.371445 26239 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:19.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.440387 26239 rpc_server.cc:307] RPC server started. Bound to: 127.25.139.125:44983
15:10:19.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.440466 26311 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.139.125:44983 every 8 connection(s)
15:10:19.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.443334 26239 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/data/info.pb
15:10:19.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.449151 26312 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.452050 26157 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 26239
15:10:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.452409 26157 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-1/wal/instance
15:10:19.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.454528 26157 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:19.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:19.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/wal
15:10:19.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/data
15:10:19.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/logs
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/data/info.pb
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.124:34107
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.124
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:19.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:19.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:19.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44893/jwks.json with env {}
15:10:19.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.460862 26312 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } has no permanent_uuid. Determining permanent_uuid...
15:10:19.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.479121 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } attempt: 7
15:10:19.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.492458 26312 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } has no permanent_uuid. Determining permanent_uuid...
15:10:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.497709 26237 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } has no permanent_uuid. Determining permanent_uuid...
15:10:19.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.508106 26312 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } has no permanent_uuid. Determining permanent_uuid...
15:10:19.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.508337 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:19.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.510587 26263 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:19.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.515539 26312 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:19.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.565032 26312 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 1
15:10:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.566756 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 1
15:10:19.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.569936 26312 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:19.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.571159 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:19.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.638381 26312 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 2
15:10:19.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.642984 26312 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:19.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.652557 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 2
15:10:19.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.656662 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:19.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.734464 26312 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 3
15:10:19.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.738663 26312 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:19.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.742053 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 3
15:10:19.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.745915 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:19.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.777419 26314 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:19.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.778056 26314 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:19.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.778676 26314 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:19.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.811271 26314 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:19.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.811626 26314 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:19.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.811925 26314 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:19.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.812199 26314 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:19.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.812481 26314 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:19.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.812696 26314 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:44893/jwks.json
15:10:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.848838 26314 master_runner.cc:386] Master server non-default flags:
15:10:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/data
15:10:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/wal
15:10:19.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.124:34107
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44893/jwks.json
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/data/info.pb
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.124
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/logs
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:19.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.850560 26314 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:19.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.852222 26314 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:19.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.863708 26327 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:19.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.882097 26312 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 4
15:10:19.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:19.886312 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 4
15:10:19.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.894809 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:19.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.896060 26312 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:20.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:20.178354 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 5
15:10:20.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:20.186160 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:20.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:20.195585 26312 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 5
15:10:20.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:20.203410 26312 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:20.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:20.738809 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 6
15:10:20.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:20.750972 26312 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 6
15:10:20.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:20.762773 26237 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:20.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:20.774493 26312 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.25.139.124:34107: Network error: Client connection negotiation failed: client connection to 127.25.139.124:34107: connect: Connection refused (error 111)
15:10:21.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:21.267340 26326 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec80 after lost signal to thread 26314
15:10:21.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:21.394987 26314 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.531s user 0.467s sys 1.055s
15:10:21.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:21.395360 26314 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.532s user 0.467s sys 1.055s
15:10:21.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:19.864208 26328 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:21.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:21.396518 26329 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1531 milliseconds
15:10:21.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:21.397800 26330 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:21.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.397732 26314 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:21.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.400924 26314 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:21.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.403426 26314 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:21.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.404801 26314 hybrid_clock.cc:648] HybridClock initialized: now 1765638621404758 us; error 50 us; skew 500 ppm
15:10:21.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.405758 26314 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:21.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.412218 26314 webserver.cc:492] Webserver started at http://127.25.139.124:37505/ using document root <none> and password file <none>
15:10:21.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.413233 26314 fs_manager.cc:362] Metadata directory not provided
15:10:21.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.413491 26314 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:21.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.413997 26314 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:10:21.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.418473 26314 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/data/instance:
15:10:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f2cd0b2eaba493d9f44f3587114dfda"
15:10:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:21 on dist-test-slave-xht5"
15:10:21.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.419673 26314 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/wal/instance:
15:10:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f2cd0b2eaba493d9f44f3587114dfda"
15:10:21.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:21 on dist-test-slave-xht5"
15:10:21.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.427412 26314 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
15:10:21.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.433367 26342 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:21.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.434576 26314 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
15:10:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.434959 26314 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/wal
15:10:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1f2cd0b2eaba493d9f44f3587114dfda"
15:10:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:21 on dist-test-slave-xht5"
15:10:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.435310 26314 fs_report.cc:389] FS layout report
15:10:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/wal
15:10:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/wal
15:10:21.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/data/data
15:10:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:21.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:21.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.501565 26314 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.503124 26314 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:21.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.503572 26314 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.583281 26314 rpc_server.cc:307] RPC server started. Bound to: 127.25.139.124:34107
15:10:21.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.583329 26393 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.139.124:34107 every 8 connection(s)
15:10:21.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.586002 26314 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/data/info.pb
15:10:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.588774 26157 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 26314
15:10:21.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.589344 26157 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/master-2/wal/instance
15:10:21.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.592739 26394 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:21.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.608816 26394 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } has no permanent_uuid. Determining permanent_uuid...
15:10:21.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.633944 26394 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } has no permanent_uuid. Determining permanent_uuid...
15:10:21.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.644872 26394 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } has no permanent_uuid. Determining permanent_uuid...
15:10:21.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.661080 26394 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda: Bootstrap starting.
15:10:21.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.667639 26394 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda: Neither blocks nor log segments found. Creating new log.
15:10:21.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.669389 26394 log.cc:826] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda: Log is configured to *not* fsync() on all Append() calls
15:10:21.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.673790 26394 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda: No bootstrap required, opened a new log
15:10:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.693506 26394 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } }
15:10:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.694177 26394 raft_consensus.cc:385] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:10:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.694396 26394 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1f2cd0b2eaba493d9f44f3587114dfda, State: Initialized, Role: FOLLOWER
15:10:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.695137 26394 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } }
15:10:21.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.697593 26402 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } } }
15:10:21.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.698593 26402 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [sys.catalog]: This master's current role is: FOLLOWER
15:10:21.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.699805 26394 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [sys.catalog]: configured and running, proceeding with master startup.
15:10:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:21.725003 26413 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:21.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:21.725412 26413 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.720628 26314 master_runner.cc:186] Error getting master registration for 127.25.139.126:46321: OK, instance_id { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" instance_seqno: 1765638617074819 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:10:21.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.743649 26314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:21.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:21.759984 26348 tablet.cc:2378] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:10:21.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.796321 26237 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 7
15:10:21.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.814785 26237 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec: Bootstrap starting.
15:10:21.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.820369 26237 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec: Neither blocks nor log segments found. Creating new log.
15:10:21.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.822194 26237 log.cc:826] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec: Log is configured to *not* fsync() on all Append() calls
15:10:21.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.828410 26237 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec: No bootstrap required, opened a new log
15:10:21.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.835062 26312 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } attempt: 7
15:10:21.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.851583 26237 raft_consensus.cc:359] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } }
15:10:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.852741 26237 raft_consensus.cc:385] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:10:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.853212 26237 raft_consensus.cc:740] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f24cb3a898fb4b04a2a2d39cadadd3ec, State: Initialized, Role: FOLLOWER
15:10:21.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.854148 26237 consensus_queue.cc:260] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } }
15:10:21.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.857067 26418 sys_catalog.cc:455] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } } }
15:10:21.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.858026 26418 sys_catalog.cc:458] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [sys.catalog]: This master's current role is: FOLLOWER
15:10:21.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.859325 26237 sys_catalog.cc:565] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [sys.catalog]: configured and running, proceeding with master startup.
15:10:21.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.861065 26312 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42: Bootstrap starting.
15:10:21.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.870595 26312 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42: Neither blocks nor log segments found. Creating new log.
15:10:21.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.875341 26312 log.cc:826] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42: Log is configured to *not* fsync() on all Append() calls
15:10:21.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.881937 26312 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42: No bootstrap required, opened a new log
15:10:21.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.882050 26402 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:21.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.882591 26402 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } }
15:10:21.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.887219 26402 leader_election.cc:290] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers f24cb3a898fb4b04a2a2d39cadadd3ec (127.25.139.126:46321), 9f24086a02154b8ba60587ba1a8d2d42 (127.25.139.125:44983)
15:10:21.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.888226 26212 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" is_pre_election: true
15:10:21.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.889374 26212 raft_consensus.cc:2468] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1f2cd0b2eaba493d9f44f3587114dfda in term 0.
15:10:21.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.891081 26346 leader_election.cc:304] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1f2cd0b2eaba493d9f44f3587114dfda, f24cb3a898fb4b04a2a2d39cadadd3ec; no voters:
15:10:21.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.892091 26402 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [term 0 FOLLOWER]: Leader pre-election won for term 1
15:10:21.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.892444 26402 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:10:21.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.892773 26402 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [term 0 FOLLOWER]: Advancing to term 1
15:10:21.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.899791 26402 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } }
15:10:21.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.906471 26402 leader_election.cc:290] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [CANDIDATE]: Term 1 election: Requested vote from peers f24cb3a898fb4b04a2a2d39cadadd3ec (127.25.139.126:46321), 9f24086a02154b8ba60587ba1a8d2d42 (127.25.139.125:44983)
15:10:21.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.907437 26212 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec"
15:10:21.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.908175 26212 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [term 0 FOLLOWER]: Advancing to term 1
15:10:21.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.914074 26430 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:10:21.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.922399 26212 raft_consensus.cc:2468] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f2cd0b2eaba493d9f44f3587114dfda in term 1.
15:10:21.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.927464 26312 raft_consensus.cc:359] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } }
15:10:21.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.928411 26312 raft_consensus.cc:385] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:10:21.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.928728 26312 raft_consensus.cc:740] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9f24086a02154b8ba60587ba1a8d2d42, State: Initialized, Role: FOLLOWER
15:10:21.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.930234 26346 leader_election.cc:304] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1f2cd0b2eaba493d9f44f3587114dfda, f24cb3a898fb4b04a2a2d39cadadd3ec; no voters:
15:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.931088 26402 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [term 1 FOLLOWER]: Leader election won for term 1
15:10:21.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.929651 26312 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } }
15:10:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.946110 26312 sys_catalog.cc:565] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [sys.catalog]: configured and running, proceeding with master startup.
15:10:21.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.946259 26287 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9f24086a02154b8ba60587ba1a8d2d42" is_pre_election: true
15:10:21.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.947201 26287 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1f2cd0b2eaba493d9f44f3587114dfda in term 0.
15:10:21.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.949115 26286 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9f24086a02154b8ba60587ba1a8d2d42"
15:10:21.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.949841 26286 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [term 0 FOLLOWER]: Advancing to term 1
15:10:21.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.957764 26402 raft_consensus.cc:697] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [term 1 LEADER]: Becoming Leader. State: Replica: 1f2cd0b2eaba493d9f44f3587114dfda, State: Running, Role: LEADER
15:10:21.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.958875 26402 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } }
15:10:21.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.959333 26286 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1f2cd0b2eaba493d9f44f3587114dfda in term 1.
15:10:21.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.944510 26431 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } } }
15:10:21.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.961828 26431 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [sys.catalog]: This master's current role is: FOLLOWER
15:10:21.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.975443 26432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1f2cd0b2eaba493d9f44f3587114dfda. Latest consensus state: current_term: 1 leader_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } } }
15:10:21.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.976265 26432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [sys.catalog]: This master's current role is: LEADER
15:10:21.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.933429 26169 master_runner.cc:186] Error getting master registration for 127.25.139.125:44983: OK, instance_id { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" instance_seqno: 1765638619421475 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:10:21.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.983057 26169 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.985764 26439 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:10:21.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:21.992938 26439 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:10:22.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.015647 26286 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [term 1 FOLLOWER]: Refusing update from remote peer 1f2cd0b2eaba493d9f44f3587114dfda: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:10:22.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.017593 26432 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [LEADER]: Connected to new peer: Peer: permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:10:22.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.018308 26212 raft_consensus.cc:1275] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [term 1 FOLLOWER]: Refusing update from remote peer 1f2cd0b2eaba493d9f44f3587114dfda: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:10:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:22.018925 26446 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:22.019369 26446 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:22.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.020709 26432 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [LEADER]: Connected to new peer: Peer: permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:10:22.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:22.034369 26191 tablet.cc:2378] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:10:22.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.056267 26418 sys_catalog.cc:455] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1f2cd0b2eaba493d9f44f3587114dfda. Latest consensus state: current_term: 1 leader_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } } }
15:10:22.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.062245 26418 sys_catalog.cc:458] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [sys.catalog]: This master's current role is: FOLLOWER
15:10:22.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.071872 26431 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1f2cd0b2eaba493d9f44f3587114dfda. Latest consensus state: current_term: 1 leader_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } } }
15:10:22.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.072609 26431 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [sys.catalog]: This master's current role is: FOLLOWER
15:10:22.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.086875 26449 mvcc.cc:204] Tried to move back new op lower bound from 7232055795759599616 to 7232055795554635776. Current Snapshot: MvccSnapshot[applied={T|T < 7232055795759599616}]
15:10:22.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.097113 26418 sys_catalog.cc:455] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } } }
15:10:22.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.097826 26418 sys_catalog.cc:458] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec [sys.catalog]: This master's current role is: FOLLOWER
15:10:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.104116 26439 catalog_manager.cc:1357] Generated new cluster ID: 5dca94eadc2e44619aa0d7ddd15bc0ee
15:10:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.104442 26439 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:10:22.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.112087 26432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } } }
15:10:22.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.112919 26432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [sys.catalog]: This master's current role is: LEADER
15:10:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.113423 26432 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } } }
15:10:22.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.114102 26432 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda [sys.catalog]: This master's current role is: LEADER
15:10:22.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.138669 26431 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "f24cb3a898fb4b04a2a2d39cadadd3ec" member_type: VOTER last_known_addr { host: "127.25.139.126" port: 46321 } } peers { permanent_uuid: "9f24086a02154b8ba60587ba1a8d2d42" member_type: VOTER last_known_addr { host: "127.25.139.125" port: 44983 } } peers { permanent_uuid: "1f2cd0b2eaba493d9f44f3587114dfda" member_type: VOTER last_known_addr { host: "127.25.139.124" port: 34107 } } }
15:10:22.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.139364 26431 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42 [sys.catalog]: This master's current role is: FOLLOWER
15:10:22.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.182600 26439 catalog_manager.cc:1380] Generated new certificate authority record
15:10:22.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.184552 26439 catalog_manager.cc:1514] Loading token signing keys...
15:10:22.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.229233 26439 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 1f2cd0b2eaba493d9f44f3587114dfda: Generated new TSK 0
15:10:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.231015 26439 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:10:22.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.249315 26157 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:22.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:22.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/wal
15:10:22.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/data
15:10:22.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:22.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:22.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/logs
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/data/info.pb
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:10:22.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:22.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.65:0
15:10:22.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.139.65
15:10:22.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.65
15:10:22.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:22.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:22.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:22.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:22.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:22.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:10:22.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:22.662958 26456 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:22.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:22.663830 26456 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:22.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:22.664739 26456 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:22.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:22.711812 26456 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:22.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:22.713047 26456 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.139.65
15:10:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.752219 26456 tablet_server_runner.cc:78] Tablet server non-default flags:
15:10:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/data
15:10:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/wal
15:10:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:22.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.65:0
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/data/info.pb
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.65
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.139.65
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/logs
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:22.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:22.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.753954 26456 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:22.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.755721 26456 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:22.769879 26462 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:22.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.946537 26430 catalog_manager.cc:1269] Loaded cluster ID: 5dca94eadc2e44619aa0d7ddd15bc0ee
15:10:22.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.947564 26430 catalog_manager.cc:1562] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec: loading cluster ID for follower catalog manager: success
15:10:22.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:22.987314 26430 catalog_manager.cc:1584] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec: acquiring CA information for follower catalog manager: success
15:10:23.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:23.011938 26430 catalog_manager.cc:1612] T 00000000000000000000000000000000 P f24cb3a898fb4b04a2a2d39cadadd3ec: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:10:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:23.027845 26446 catalog_manager.cc:1269] Loaded cluster ID: 5dca94eadc2e44619aa0d7ddd15bc0ee
15:10:23.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:23.028506 26446 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42: loading cluster ID for follower catalog manager: success
15:10:23.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:23.055495 26446 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42: acquiring CA information for follower catalog manager: success
15:10:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:23.080680 26446 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 9f24086a02154b8ba60587ba1a8d2d42: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:10:25.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:24.172313 26461 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 26456
15:10:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:25.352289 26461 kernel_stack_watchdog.cc:198] Thread 26456 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 396ms:
15:10:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:10:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:10:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:10:25.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:10:25.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:25.353927 26465 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:25.356134 26456 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.582s user 0.554s sys 1.114s
15:10:25.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:25.356494 26456 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.583s user 0.554s sys 1.114s
15:10:25.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:22.773978 26463 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:25.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:25.360963 26464 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2586 milliseconds
15:10:25.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.361263 26456 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:25.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.363698 26456 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:25.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.372241 26456 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:25.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.373903 26456 hybrid_clock.cc:648] HybridClock initialized: now 1765638625373869 us; error 49 us; skew 500 ppm
15:10:25.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.375134 26456 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:25.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.394738 26456 webserver.cc:492] Webserver started at http://127.25.139.65:45113/ using document root <none> and password file <none>
15:10:25.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.396275 26456 fs_manager.cc:362] Metadata directory not provided
15:10:25.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.396728 26456 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:25.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.397394 26456 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:10:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.403961 26456 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/data/instance:
15:10:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1019644d9a3c4996b3d405889db4f78f"
15:10:25.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:25 on dist-test-slave-xht5"
15:10:25.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.405784 26456 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/wal/instance:
15:10:25.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1019644d9a3c4996b3d405889db4f78f"
15:10:25.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:25 on dist-test-slave-xht5"
15:10:25.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.421249 26456 fs_manager.cc:696] Time spent creating directory manager: real 0.014s user 0.013s sys 0.000s
15:10:25.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.434916 26476 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:25.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.436583 26456 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.009s sys 0.003s
15:10:25.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.437124 26456 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/wal
15:10:25.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1019644d9a3c4996b3d405889db4f78f"
15:10:25.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:25 on dist-test-slave-xht5"
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.437708 26456 fs_report.cc:389] FS layout report
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/wal
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/wal
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/data/data
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:25.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.539448 26456 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:25.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.541937 26456 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:25.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.542855 26456 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:25.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.552654 26456 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:10:25.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.566097 26456 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:10:25.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.566634 26456 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.001s
15:10:25.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.567039 26456 ts_tablet_manager.cc:616] Registered 0 tablets
15:10:25.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.567343 26456 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:10:25.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.814286 26456 rpc_server.cc:307] RPC server started. Bound to: 127.25.139.65:32873
15:10:25.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.814429 26590 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.139.65:32873 every 8 connection(s)
15:10:25.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.818145 26456 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/data/info.pb
15:10:25.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.820675 26157 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 26456
15:10:25.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.821331 26157 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-0/wal/instance
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.837749 26157 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/wal
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/data
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/logs
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/data/info.pb
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.66:0
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.139.66
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.66
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:10:25.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.922528 26591 heartbeater.cc:344] Connected to a master server at 127.25.139.124:34107
15:10:25.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.922976 26591 heartbeater.cc:461] Registering TS with master...
15:10:25.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.924296 26591 heartbeater.cc:507] Master 127.25.139.124:34107 requested a full tablet report, sending...
15:10:25.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.927803 26359 ts_manager.cc:194] Registered new tserver with Master: 1019644d9a3c4996b3d405889db4f78f (127.25.139.65:32873)
15:10:25.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.933588 26359 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.25.139.65:50941
15:10:25.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.948380 26593 heartbeater.cc:344] Connected to a master server at 127.25.139.125:44983
15:10:25.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.948776 26593 heartbeater.cc:461] Registering TS with master...
15:10:25.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.949433 26593 heartbeater.cc:507] Master 127.25.139.125:44983 requested a full tablet report, sending...
15:10:25.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.951650 26596 heartbeater.cc:344] Connected to a master server at 127.25.139.126:46321
15:10:25.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.952023 26596 heartbeater.cc:461] Registering TS with master...
15:10:25.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.952561 26277 ts_manager.cc:194] Registered new tserver with Master: 1019644d9a3c4996b3d405889db4f78f (127.25.139.65:32873)
15:10:25.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.952782 26596 heartbeater.cc:507] Master 127.25.139.126:46321 requested a full tablet report, sending...
15:10:25.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:25.957337 26202 ts_manager.cc:194] Registered new tserver with Master: 1019644d9a3c4996b3d405889db4f78f (127.25.139.65:32873)
15:10:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:26.373711 26592 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:26.374220 26592 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:26.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:26.374739 26592 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:26.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:26.407903 26592 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:26.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:26.409003 26592 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.139.66
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:26.446614 26592 tablet_server_runner.cc:78] Tablet server non-default flags:
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/data
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/wal
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.66:0
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/data/info.pb
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.66
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.139.66
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/logs
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:26.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:26.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:26.448302 26592 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:26.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:26.450429 26592 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:26.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:26.465816 26607 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:26.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:26.938918 26591 heartbeater.cc:499] Master 127.25.139.124:34107 was elected leader, sending a full tablet report...
15:10:28.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:27.868253 26606 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 26592
15:10:28.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:28.160143 26592 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.685s user 0.549s sys 1.038s
15:10:28.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:28.160624 26592 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.686s user 0.549s sys 1.038s
15:10:28.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:26.477295 26608 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:28.161787 26609 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1685 milliseconds
15:10:28.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:28.163120 26610 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:28.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.163079 26592 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:28.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.166433 26592 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:28.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.168569 26592 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:28.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.169920 26592 hybrid_clock.cc:648] HybridClock initialized: now 1765638628169894 us; error 43 us; skew 500 ppm
15:10:28.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.170781 26592 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:28.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.176832 26592 webserver.cc:492] Webserver started at http://127.25.139.66:43527/ using document root <none> and password file <none>
15:10:28.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.177762 26592 fs_manager.cc:362] Metadata directory not provided
15:10:28.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.177979 26592 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:28.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.178503 26592 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:10:28.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.182914 26592 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/data/instance:
15:10:28.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "75c246133fe0409b824127b16253ec47"
15:10:28.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:28 on dist-test-slave-xht5"
15:10:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.184142 26592 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/wal/instance:
15:10:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "75c246133fe0409b824127b16253ec47"
15:10:28.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:28 on dist-test-slave-xht5"
15:10:28.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.191462 26592 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
15:10:28.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.197268 26618 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:28.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.198343 26592 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.001s
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.198750 26592 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/wal
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "75c246133fe0409b824127b16253ec47"
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:28 on dist-test-slave-xht5"
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.199084 26592 fs_report.cc:389] FS layout report
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/wal
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/wal
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/data/data
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:28.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:28.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.248972 26592 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:28.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.250602 26592 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:28.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.251088 26592 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:28.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.253818 26592 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:10:28.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.258716 26592 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:10:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.258939 26592 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:10:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.259260 26592 ts_tablet_manager.cc:616] Registered 0 tablets
15:10:28.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.259431 26592 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:10:28.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.397902 26592 rpc_server.cc:307] RPC server started. Bound to: 127.25.139.66:33989
15:10:28.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.398017 26730 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.139.66:33989 every 8 connection(s)
15:10:28.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.400594 26592 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/data/info.pb
15:10:28.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.409411 26157 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 26592
15:10:28.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.409889 26157 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-1/wal/instance
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.433939 26157 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/wal
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/data
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/logs
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/data/info.pb
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.67:0
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.139.67
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.67
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:28.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:28.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:28.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:28.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:28.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
15:10:28.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.445907 26733 heartbeater.cc:344] Connected to a master server at 127.25.139.126:46321
15:10:28.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.447058 26733 heartbeater.cc:461] Registering TS with master...
15:10:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.448393 26731 heartbeater.cc:344] Connected to a master server at 127.25.139.124:34107
15:10:28.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.448733 26731 heartbeater.cc:461] Registering TS with master...
15:10:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.448701 26733 heartbeater.cc:507] Master 127.25.139.126:46321 requested a full tablet report, sending...
15:10:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.449007 26732 heartbeater.cc:344] Connected to a master server at 127.25.139.125:44983
15:10:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.449414 26732 heartbeater.cc:461] Registering TS with master...
15:10:28.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.449689 26731 heartbeater.cc:507] Master 127.25.139.124:34107 requested a full tablet report, sending...
15:10:28.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.450328 26732 heartbeater.cc:507] Master 127.25.139.125:44983 requested a full tablet report, sending...
15:10:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.451910 26359 ts_manager.cc:194] Registered new tserver with Master: 75c246133fe0409b824127b16253ec47 (127.25.139.66:33989)
15:10:28.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.452041 26202 ts_manager.cc:194] Registered new tserver with Master: 75c246133fe0409b824127b16253ec47 (127.25.139.66:33989)
15:10:28.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.453676 26359 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.25.139.66:55489
15:10:28.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.455345 26277 ts_manager.cc:194] Registered new tserver with Master: 75c246133fe0409b824127b16253ec47 (127.25.139.66:33989)
15:10:28.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:28.765645 26741 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:28.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:28.766145 26741 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:28.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:28.766646 26741 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:28.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:28.799026 26741 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:28.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:28.799837 26741 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.25.139.67
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.844887 26741 tablet_server_runner.cc:78] Tablet server non-default flags:
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.25.139.84:45711
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/data
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/wal
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.25.139.67:0
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/data/info.pb
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.25.139.67
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.25.139.126:46321,127.25.139.125:44983,127.25.139.124:34107
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:28.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.25.139.67
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/logs
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:28.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.846148 26741 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:28.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:28.847806 26741 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:28.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:28.860160 26747 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:29.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:29.457784 26731 heartbeater.cc:499] Master 127.25.139.124:34107 was elected leader, sending a full tablet report...
15:10:30.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:28.860874 26748 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:30.127839 26749 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1266 milliseconds
15:10:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:30.128153 26750 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.128165 26741 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:30.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.131731 26741 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:30.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.134012 26741 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:30.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.135393 26741 hybrid_clock.cc:648] HybridClock initialized: now 1765638630135358 us; error 53 us; skew 500 ppm
15:10:30.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.136209 26741 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:30.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.142832 26741 webserver.cc:492] Webserver started at http://127.25.139.67:43073/ using document root <none> and password file <none>
15:10:30.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.143765 26741 fs_manager.cc:362] Metadata directory not provided
15:10:30.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.143956 26741 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:30.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.144366 26741 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:10:30.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.148931 26741 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/data/instance:
15:10:30.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bde5c6bdc92c4db19b90b032fbca4766"
15:10:30.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:30 on dist-test-slave-xht5"
15:10:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.150054 26741 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/wal/instance:
15:10:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bde5c6bdc92c4db19b90b032fbca4766"
15:10:30.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:30 on dist-test-slave-xht5"
15:10:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.157806 26741 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.004s
15:10:30.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.163736 26758 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:30.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.164932 26741 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
15:10:30.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.165280 26741 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/wal
15:10:30.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "bde5c6bdc92c4db19b90b032fbca4766"
15:10:30.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:30 on dist-test-slave-xht5"
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.165589 26741 fs_report.cc:389] FS layout report
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/wal
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/wal
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/data/data
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:30.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:30.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.226470 26741 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:30.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.228137 26741 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:30.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.228593 26741 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:30.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.231467 26741 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
15:10:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.235889 26741 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
15:10:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.236097 26741 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
15:10:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.236311 26741 ts_tablet_manager.cc:616] Registered 0 tablets
15:10:30.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.236446 26741 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
15:10:30.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.375360 26741 rpc_server.cc:307] RPC server started. Bound to: 127.25.139.67:37493
15:10:30.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.375396 26870 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.25.139.67:37493 every 8 connection(s)
15:10:30.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.378038 26741 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/data/info.pb
15:10:30.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.380852 26157 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 26741
15:10:30.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.381383 26157 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster911753638959418607/ts-2/wal/instance
15:10:30.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.417855 26873 heartbeater.cc:344] Connected to a master server at 127.25.139.126:46321
15:10:30.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.418444 26873 heartbeater.cc:461] Registering TS with master...
15:10:30.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.419860 26873 heartbeater.cc:507] Master 127.25.139.126:46321 requested a full tablet report, sending...
15:10:30.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.421581 26872 heartbeater.cc:344] Connected to a master server at 127.25.139.125:44983
15:10:30.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.421895 26872 heartbeater.cc:461] Registering TS with master...
15:10:30.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.422884 26872 heartbeater.cc:507] Master 127.25.139.125:44983 requested a full tablet report, sending...
15:10:30.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.423408 26202 ts_manager.cc:194] Registered new tserver with Master: bde5c6bdc92c4db19b90b032fbca4766 (127.25.139.67:37493)
15:10:30.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.425812 26277 ts_manager.cc:194] Registered new tserver with Master: bde5c6bdc92c4db19b90b032fbca4766 (127.25.139.67:37493)
15:10:30.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.428766 26871 heartbeater.cc:344] Connected to a master server at 127.25.139.124:34107
15:10:30.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.429119 26871 heartbeater.cc:461] Registering TS with master...
15:10:30.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.429952 26871 heartbeater.cc:507] Master 127.25.139.124:34107 requested a full tablet report, sending...
15:10:30.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.431939 26359 ts_manager.cc:194] Registered new tserver with Master: bde5c6bdc92c4db19b90b032fbca4766 (127.25.139.67:37493)
15:10:30.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.433679 26359 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.25.139.67:57901
15:10:30.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.448212 26157 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
15:10:30.459 [DEBUG - main] (MiniKuduCluster.java:195) Response:
15:10:30.459 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
15:10:30.460 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.25.139.126"
port: 46321
}
bound_http_address {
host: "127.25.139.126"
port: 34901
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.25.139.125"
port: 44983
}
bound_http_address {
host: "127.25.139.125"
port: 37941
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.25.139.124"
port: 34107
}
bound_http_address {
host: "127.25.139.124"
port: 37505
}
}
}
15:10:30.460 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:10:30.461 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.25.139.65"
port: 32873
}
bound_http_address {
host: "127.25.139.65"
port: 45113
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.25.139.66"
port: 33989
}
bound_http_address {
host: "127.25.139.66"
port: 43527
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.25.139.67"
port: 37493
}
bound_http_address {
host: "127.25.139.67"
port: 43073
}
}
}
15:10:30.467 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
15:10:30.470 [DEBUG - main] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster2853318218922810624/krb5kdc/krb5cc
15:10:30.470 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:10:30.472 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskrL_IJG/test-tmp
15:10:30.472 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
15:10:30.473 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskrL_IJG/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
15:10:30.476 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 0
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
15:10:30.828 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:30.828 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
15:10:30.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.831215 26882 mini_oidc.cc:150] Starting JWKS server
15:10:30.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.834437 26882 webserver.cc:492] Webserver started at http://127.0.0.1:36497/ using document root <none> and password file <none>
15:10:30.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.834898 26882 mini_oidc.cc:186] Starting OIDC Discovery server
15:10:30.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:30.836542 26882 webserver.cc:492] Webserver started at http://127.0.0.1:46011/ using document root <none> and password file <none>
15:10:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
15:10:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/principal' for realm 'KRBTEST.COM',
15:10:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
15:10:30.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:30 dist-test-slave-xht5 krb5kdc[26893](info): setting up network...
15:10:30.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
15:10:30.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:30 dist-test-slave-xht5 krb5kdc[26893](info): set up 2 sockets
15:10:30.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:30 dist-test-slave-xht5 krb5kdc[26893](info): commencing operation
15:10:30.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
15:10:31.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:31.437114 26871 heartbeater.cc:499] Master 127.25.139.124:34107 was elected leader, sending a full tablet report...
15:10:32.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:32.949878 26882 mini_kdc.cc:121] Time spent starting KDC: real 2.076s user 0.003s sys 0.016s
15:10:32.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
15:10:32.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:10:32.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
15:10:32.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
15:10:33.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:10:33.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
15:10:33.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
15:10:33.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:10:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
15:10:33.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
15:10:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/test-user.keytab.
15:10:33.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/test-user.keytab.
15:10:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:33 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638633, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:33.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:10:33.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:10:33Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
15:10:33.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:10:33Z Disabled control of system clock
15:10:33.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.26.64.190@KRBTEST.COM; defaulting to no policy
15:10:33.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:33.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.26.64.190@KRBTEST.COM" created.
15:10:33.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:33.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.64.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:33.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.64.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:33.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.26.64.190@KRBTEST.COM; defaulting to no policy
15:10:33.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:33.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.26.64.190@KRBTEST.COM" created.
15:10:33.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:33.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.26.64.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:33.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.26.64.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:33.184657 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/logs
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.190:36655
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.190
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.190
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:10:33.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:33.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.508072 26910 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:33.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.508759 26910 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:33.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.509512 26910 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.542124 26910 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:33.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.542438 26910 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:33.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.542836 26910 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:33.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.543119 26910 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:33.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.543401 26910 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:33.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.543673 26910 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:33.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.543887 26910 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:10:33.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:33.582136 26910 master_runner.cc:386] Master server non-default flags:
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.190
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.190:36655
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:33.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.190
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/logs
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:33.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:33.583931 26910 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:33.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:33.585227 26910 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:33.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.597620 26916 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:35.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:33.598397 26917 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:35.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.002069 26915 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 26910
15:10:35.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.388222 26915 kernel_stack_watchdog.cc:198] Thread 26910 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
15:10:35.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:10:35.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:10:35.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:10:35.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:10:35.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.388654 26910 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.791s user 0.599s sys 1.168s
15:10:35.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.389038 26910 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.791s user 0.599s sys 1.168s
15:10:35.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.392890 26918 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1789 milliseconds
15:10:35.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.394275 26919 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:35.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.394356 26910 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:35.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.395550 26910 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:35.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.398138 26910 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:35.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.399503 26910 hybrid_clock.cc:648] HybridClock initialized: now 1765638635399466 us; error 43 us; skew 500 ppm
15:10:35.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.400514 26910 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:35.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:35 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638635, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:35.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.407013 26910 init.cc:377] Logged in from keytab as kudu/127.26.64.190@KRBTEST.COM (short username kudu)
15:10:35.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.412199 26910 webserver.cc:492] Webserver started at http://127.26.64.190:44727/ using document root <none> and password file <none>
15:10:35.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.413146 26910 fs_manager.cc:362] Metadata directory not provided
15:10:35.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.413368 26910 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:35.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.413826 26910 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:10:35.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.418628 26910 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/instance:
15:10:35.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86daac6df1334b1bb65f5108dcdd44cc"
15:10:35.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:35 on dist-test-slave-xht5"
15:10:35.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.419926 26910 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal/instance:
15:10:35.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86daac6df1334b1bb65f5108dcdd44cc"
15:10:35.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:35 on dist-test-slave-xht5"
15:10:35.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.427467 26910 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
15:10:35.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.432615 26930 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:35.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.433650 26910 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.001s sys 0.005s
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.433960 26910 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86daac6df1334b1bb65f5108dcdd44cc"
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:35 on dist-test-slave-xht5"
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.434285 26910 fs_report.cc:389] FS layout report
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/data
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:35.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:35.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:35.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:35.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:35.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.488426 26910 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:35.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.499387 26910 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:35.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.499898 26910 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.571226 26910 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.190:36655
15:10:35.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.571295 26981 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.190:36655 every 8 connection(s)
15:10:35.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.574106 26910 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:10:35.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.578617 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 26910
15:10:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.579150 26882 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal/instance
15:10:35.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.580901 26982 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:35.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.594300 26982 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } has no permanent_uuid. Determining permanent_uuid...
15:10:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.26.64.189@KRBTEST.COM; defaulting to no policy
15:10:35.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.26.64.189@KRBTEST.COM" created.
15:10:35.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:35 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638635, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:10:35.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:35.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.64.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:35.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.64.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.635506 26982 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } has no permanent_uuid. Determining permanent_uuid...
15:10:35.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.637669 26932 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:35.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.26.64.189@KRBTEST.COM; defaulting to no policy
15:10:35.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.641662 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:35.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:35.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.26.64.189@KRBTEST.COM" created.
15:10:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.26.64.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:35.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.26.64.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:35.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.663568 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/logs
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.189:37765
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.189
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.189
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:10:35.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:35.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.691113 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } attempt: 1
15:10:35.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.695216 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:35.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.763674 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } attempt: 2
15:10:35.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.768028 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:35.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:35.859431 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } attempt: 3
15:10:35.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.863374 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:35.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.974238 26991 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.974920 26991 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:35.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:35.975760 26991 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:36.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:36.006808 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } attempt: 4
15:10:36.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.010568 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:36.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.011878 26991 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:36.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.012112 26991 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:36.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.012393 26991 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:36.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.012600 26991 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:36.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.012794 26991 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:36.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.013026 26991 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:36.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.013190 26991 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:36.051707 26991 master_runner.cc:386] Master server non-default flags:
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.189
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.189:37765
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.189
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/logs
15:10:36.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:36.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:36.052953 26991 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:36.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:36.053915 26991 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:36.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.064693 26998 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:36.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:36.310005 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } attempt: 5
15:10:36.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.317111 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:36.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:36.864681 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } attempt: 6
15:10:36.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.890888 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:37.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:36.065459 26999 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:37.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:37.468811 26997 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 26991
15:10:37.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:37.666694 26991 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.601s user 0.586s sys 0.988s
15:10:37.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:37.668205 26991 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.603s user 0.586s sys 0.988s
15:10:37.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:37.668771 27001 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:37.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:37.672408 27000 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1601 milliseconds
15:10:37.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.672430 26991 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:37.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.673787 26991 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:37.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.676123 26991 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:37.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.677464 26991 hybrid_clock.cc:648] HybridClock initialized: now 1765638637677423 us; error 42 us; skew 500 ppm
15:10:37.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.678298 26991 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:37.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:37 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638637, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:37.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.684551 26991 init.cc:377] Logged in from keytab as kudu/127.26.64.189@KRBTEST.COM (short username kudu)
15:10:37.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.689522 26991 webserver.cc:492] Webserver started at http://127.26.64.189:45529/ using document root <none> and password file <none>
15:10:37.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.690487 26991 fs_manager.cc:362] Metadata directory not provided
15:10:37.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.690717 26991 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:37.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.691181 26991 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:10:37.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.695361 26991 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/instance:
15:10:37.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6eade2b7fd24c10aae109da5e2e6718"
15:10:37.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:37 on dist-test-slave-xht5"
15:10:37.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.696458 26991 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal/instance:
15:10:37.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6eade2b7fd24c10aae109da5e2e6718"
15:10:37.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:37 on dist-test-slave-xht5"
15:10:37.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.703835 26991 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.000s sys 0.008s
15:10:37.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.709719 27015 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:37.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.710747 26991 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
15:10:37.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.711087 26991 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6eade2b7fd24c10aae109da5e2e6718"
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:37 on dist-test-slave-xht5"
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.711416 26991 fs_report.cc:389] FS layout report
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/data
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:37.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.765301 26991 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.770778 26991 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:37.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.771256 26991 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:37.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.845254 26991 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.189:37765
15:10:37.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.845292 27066 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.189:37765 every 8 connection(s)
15:10:37.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.848037 26991 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:10:37.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.853593 27067 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:37.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.856530 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 26991
15:10:37.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.857024 26882 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal/instance
15:10:37.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.867799 27067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } has no permanent_uuid. Determining permanent_uuid...
15:10:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.26.64.188@KRBTEST.COM; defaulting to no policy
15:10:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:37.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.26.64.188@KRBTEST.COM" created.
15:10:37.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:37 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638637, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:10:37.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:37.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.64.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:37.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.26.64.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:37.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.908387 27067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } has no permanent_uuid. Determining permanent_uuid...
15:10:37.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.26.64.188@KRBTEST.COM; defaulting to no policy
15:10:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:37.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.26.64.188@KRBTEST.COM" created.
15:10:37.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:37 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638637, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for kudu/127.26.64.189@KRBTEST.COM
15:10:37.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.933554 27067 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } has no permanent_uuid. Determining permanent_uuid...
15:10:37.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:37.935766 27017 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
15:10:37.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.26.64.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.26.64.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab.
15:10:37.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:37.939812 27067 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.941900 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/logs
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:37.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.188:32887
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.188
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.188
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:10:37.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:37.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.951380 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } attempt: 7
15:10:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:37 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638635, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for kudu/127.26.64.189@KRBTEST.COM
15:10:37.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.974659 26982 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } has no permanent_uuid. Determining permanent_uuid...
15:10:37.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:37.980244 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:37.989300 27067 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 1
15:10:37.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:37.993507 27067 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:38.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.038702 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 1
15:10:38.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.042660 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.061892 27067 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 2
15:10:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.065698 27067 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:38.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.124064 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 2
15:10:38.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.128304 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:38.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.157052 27067 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 3
15:10:38.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.160706 27067 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:38.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.213686 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 3
15:10:38.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.217737 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.258838 27077 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:38.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.259390 27077 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:38.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.260021 27077 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:38.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.292166 27077 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:38.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.292447 27077 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:38.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.292725 27077 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:38.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.292956 27077 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:38.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.293155 27077 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:38.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.293355 27077 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:38.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.293502 27077 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:10:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.304198 27067 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 4
15:10:38.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.308276 27067 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.332451 27077 master_runner.cc:386] Master server non-default flags:
15:10:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data
15:10:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.188
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.188:32887
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.188
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/logs
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.333773 27077 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.334842 27077 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:38.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.345345 27085 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:38.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.358139 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 4
15:10:38.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.363356 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:38.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.607623 27067 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 5
15:10:38.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.613791 27067 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:38.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:38.646733 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 5
15:10:38.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.652251 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:39.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.161440 27067 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 6
15:10:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:39.173381 27067 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:39.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.204694 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 6
15:10:39.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:39.214740 26982 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.64.188:32887: Network error: Client connection negotiation failed: client connection to 127.26.64.188:32887: connect: Connection refused (error 111)
15:10:39.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:38.347211 27086 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:39.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:39.665748 27088 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:39.667451 27087 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1316 milliseconds
15:10:39.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.667621 27077 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:39.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.668864 27077 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:39.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.671535 27077 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:39.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.673098 27077 hybrid_clock.cc:648] HybridClock initialized: now 1765638639673056 us; error 45 us; skew 500 ppm
15:10:39.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.673941 27077 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:39.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:39 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:39.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.680342 27077 init.cc:377] Logged in from keytab as kudu/127.26.64.188@KRBTEST.COM (short username kudu)
15:10:39.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.685905 27077 webserver.cc:492] Webserver started at http://127.26.64.188:37067/ using document root <none> and password file <none>
15:10:39.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.686985 27077 fs_manager.cc:362] Metadata directory not provided
15:10:39.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.687224 27077 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:39.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.687740 27077 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
15:10:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.692054 27077 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/instance:
15:10:39.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65b363a0f05e4d7fa201cbc9853bb322"
15:10:39.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:39 on dist-test-slave-xht5"
15:10:39.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.693372 27077 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal/instance:
15:10:39.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65b363a0f05e4d7fa201cbc9853bb322"
15:10:39.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:39 on dist-test-slave-xht5"
15:10:39.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.700816 27077 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.008s
15:10:39.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.706562 27102 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:39.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.707540 27077 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.707865 27077 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65b363a0f05e4d7fa201cbc9853bb322"
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:39 on dist-test-slave-xht5"
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.708177 27077 fs_report.cc:389] FS layout report
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/data
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:39.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:39.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:39.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:39.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:39.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.774366 27077 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:39.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.779569 27077 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:39.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.780028 27077 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:39.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.858493 27077 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.188:32887
15:10:39.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.858570 27154 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.188:32887 every 8 connection(s)
15:10:39.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.861119 27077 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:10:39.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.866600 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 27077
15:10:39.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.866698 27155 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
15:10:39.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.866940 26882 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal/instance
15:10:39.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.879868 27155 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } has no permanent_uuid. Determining permanent_uuid...
15:10:39.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:39 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638633, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.64.188@KRBTEST.COM
15:10:39.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:39 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.188@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:10:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.912127 27155 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } has no permanent_uuid. Determining permanent_uuid...
15:10:39.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:39 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.188@KRBTEST.COM for kudu/127.26.64.189@KRBTEST.COM
15:10:39.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:39 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638633, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.64.189@KRBTEST.COM
15:10:39.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.933760 27155 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } has no permanent_uuid. Determining permanent_uuid...
15:10:39.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:39 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.188@KRBTEST.COM for kudu/127.26.64.188@KRBTEST.COM
15:10:39.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:39 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638633, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:10:39.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.963259 27155 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap starting.
15:10:39.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.965895 26882 external_mini_cluster.cc:949] 0 TS(s) registered with all masters
15:10:39.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.970255 27155 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Neither blocks nor log segments found. Creating new log.
15:10:39.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.972486 27155 log.cc:826] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Log is configured to *not* fsync() on all Append() calls
15:10:39.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:39.977 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
15:10:39.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.977501 27155 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: No bootstrap required, opened a new log
15:10:39.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
15:10:39.978 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
15:10:39.979 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.26.64.190"
port: 36655
}
bound_http_address {
host: "127.26.64.190"
port: 44727
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.26.64.189"
port: 37765
}
bound_http_address {
host: "127.26.64.189"
port: 45529
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.26.64.188"
port: 32887
}
bound_http_address {
host: "127.26.64.188"
port: 37067
}
}
}
15:10:39.979 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
15:10:39.980 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:39.980 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:10:39.980 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:10:39.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.995589 27155 raft_consensus.cc:359] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:39.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.996178 27155 raft_consensus.cc:385] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:10:39.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.996366 27155 raft_consensus.cc:740] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 65b363a0f05e4d7fa201cbc9853bb322, State: Initialized, Role: FOLLOWER
15:10:39.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.997020 27155 consensus_queue.cc:260] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:39.999871 27166 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:40.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.000809 27166 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: FOLLOWER
15:10:40.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.002076 27155 sys_catalog.cc:565] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: configured and running, proceeding with master startup.
15:10:40.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:39 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:40.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:10:40.011 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:40.012 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc
15:10:40.013 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:10:40.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:40.027469 27177 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:40.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:40.027735 27177 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:40.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:40.034894 27108 tablet.cc:2378] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:10:40.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.024729 27077 master_runner.cc:186] Error getting master registration for 127.26.64.190:36655: OK, instance_id { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" instance_seqno: 1765638635551698 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:10:40.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.048024 27077 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:40.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.233913 27067 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 7
15:10:40.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:40 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638637, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for kudu/127.26.64.188@KRBTEST.COM
15:10:40.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.248270 26982 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } attempt: 7
15:10:40.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.265314 27067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Bootstrap starting.
15:10:40.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:40 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638635, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for kudu/127.26.64.188@KRBTEST.COM
15:10:40.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.272042 27067 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Neither blocks nor log segments found. Creating new log.
15:10:40.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.274263 27067 log.cc:826] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Log is configured to *not* fsync() on all Append() calls
15:10:40.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.279750 27067 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: No bootstrap required, opened a new log
15:10:40.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.292486 26982 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap starting.
15:10:40.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.301113 26982 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Neither blocks nor log segments found. Creating new log.
15:10:40.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.303757 26982 log.cc:826] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Log is configured to *not* fsync() on all Append() calls
15:10:40.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.307132 27067 raft_consensus.cc:359] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.307942 27067 raft_consensus.cc:385] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:10:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.308256 27067 raft_consensus.cc:740] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b6eade2b7fd24c10aae109da5e2e6718, State: Initialized, Role: FOLLOWER
15:10:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.309537 26982 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: No bootstrap required, opened a new log
15:10:40.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.309222 27067 consensus_queue.cc:260] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:40.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.312436 27184 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:40.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.313246 27184 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:10:40.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.314482 27067 sys_catalog.cc:565] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: configured and running, proceeding with master startup.
15:10:40.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.343091 26982 raft_consensus.cc:359] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:40.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.344147 26982 raft_consensus.cc:385] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
15:10:40.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.346050 26982 raft_consensus.cc:740] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86daac6df1334b1bb65f5108dcdd44cc, State: Initialized, Role: FOLLOWER
15:10:40.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.347254 26982 consensus_queue.cc:260] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.362498 26982 sys_catalog.cc:565] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: configured and running, proceeding with master startup.
15:10:40.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:40.364135 27195 catalog_manager.cc:1568] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:40.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.359979 27197 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:40.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:40.364434 27195 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:40.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.364755 27197 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:10:40.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.338263 26991 master_runner.cc:186] Error getting master registration for 127.26.64.190:36655: OK, instance_id { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" instance_seqno: 1765638635551698 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
15:10:40.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.372021 26991 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.405014 27166 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:40.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.405577 27166 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.409668 26957 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "65b363a0f05e4d7fa201cbc9853bb322" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "86daac6df1334b1bb65f5108dcdd44cc" is_pre_election: true
15:10:40.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.410547 26957 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 65b363a0f05e4d7fa201cbc9853bb322 in term 0.
15:10:40.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.412163 27104 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322, 86daac6df1334b1bb65f5108dcdd44cc; no voters:
15:10:40.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.414479 27166 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:10:40.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.415045 27166 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 0 FOLLOWER]: Leader pre-election won for term 1
15:10:40.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.415468 27166 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:10:40.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.415889 27166 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 0 FOLLOWER]: Advancing to term 1
15:10:40.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.428593 27166 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:40.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.433595 27042 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "65b363a0f05e4d7fa201cbc9853bb322" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b6eade2b7fd24c10aae109da5e2e6718" is_pre_election: true
15:10:40.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.434518 27042 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 65b363a0f05e4d7fa201cbc9853bb322 in term 0.
15:10:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.440384 26957 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "65b363a0f05e4d7fa201cbc9853bb322" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "86daac6df1334b1bb65f5108dcdd44cc"
15:10:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.440941 26957 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 0 FOLLOWER]: Advancing to term 1
15:10:40.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.448537 26957 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 65b363a0f05e4d7fa201cbc9853bb322 in term 1.
15:10:40.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.449687 27104 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322, 86daac6df1334b1bb65f5108dcdd44cc; no voters:
15:10:40.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.450788 27205 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Leader election won for term 1
15:10:40.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.453207 27166 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 1 election: Requested vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:10:40.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.454603 27042 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "65b363a0f05e4d7fa201cbc9853bb322" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b6eade2b7fd24c10aae109da5e2e6718"
15:10:40.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.455152 27042 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 0 FOLLOWER]: Advancing to term 1
15:10:40.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.462399 27042 raft_consensus.cc:2468] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 65b363a0f05e4d7fa201cbc9853bb322 in term 1.
15:10:40.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.496873 27205 raft_consensus.cc:697] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 LEADER]: Becoming Leader. State: Replica: 65b363a0f05e4d7fa201cbc9853bb322, State: Running, Role: LEADER
15:10:40.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.498054 27205 consensus_queue.cc:237] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:40.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:40.514910 26936 tablet.cc:2378] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:10:40.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:40.526022 27210 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
15:10:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:40.526362 27210 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
15:10:40.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:40.529402 27021 tablet.cc:2378] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Can't schedule compaction. Clean time has not been advanced past its initial value.
15:10:40.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.544099 27166 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 65b363a0f05e4d7fa201cbc9853bb322. Latest consensus state: current_term: 1 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:40.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.545080 27166 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: LEADER
15:10:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.558871 27213 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:10:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.565590 27213 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:10:40.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.611404 27042 raft_consensus.cc:1275] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 1 FOLLOWER]: Refusing update from remote peer 65b363a0f05e4d7fa201cbc9853bb322: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:10:40.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.615252 27166 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:10:40.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.614207 26957 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 1 FOLLOWER]: Refusing update from remote peer 65b363a0f05e4d7fa201cbc9853bb322: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
15:10:40.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.619315 27166 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [LEADER]: Connected to new peer: Peer: permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
15:10:40.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.655133 27214 mvcc.cc:204] Tried to move back new op lower bound from 7232055871928516608 to 7232055871495737344. Current Snapshot: MvccSnapshot[applied={T|T < 7232055871928516608}]
15:10:40.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.658535 27197 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 65b363a0f05e4d7fa201cbc9853bb322. Latest consensus state: current_term: 1 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:40.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.659828 27197 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:10:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.684371 27184 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 65b363a0f05e4d7fa201cbc9853bb322. Latest consensus state: current_term: 1 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:40.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.685240 27184 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:10:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.690722 27205 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:40.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.691422 27205 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: LEADER
15:10:40.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.699591 27213 catalog_manager.cc:1357] Generated new cluster ID: 369b7cafdbc546759fa24083ca431298
15:10:40.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.699944 27213 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:10:40.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.702387 27205 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:40.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.703023 27205 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: LEADER
15:10:40.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.707700 27216 mvcc.cc:204] Tried to move back new op lower bound from 7232055871928516608 to 7232055871495737344. Current Snapshot: MvccSnapshot[applied={T|T < 7232055871928516608}]
15:10:40.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.707695 27197 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:40.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.708345 27197 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:10:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.713973 27184 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.714682 27184 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:10:40.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.756398 27213 catalog_manager.cc:1380] Generated new certificate authority record
15:10:40.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.758318 27213 catalog_manager.cc:1514] Loading token signing keys...
15:10:40.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.803069 27213 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Generated new TSK 0
15:10:40.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:40.804209 27213 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:10:41.014 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.188:32887
15:10:41.015 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:10:41.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:41.015772 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 27077
15:10:41.045 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:41.046 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.190:36655
15:10:41.046 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:10:41.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:41.046514 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 26910
15:10:41.076 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:41.076 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.189:37765
15:10:41.076 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:10:41.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:41.076640 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 26991
15:10:41.108 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:41.108 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.188:32887
15:10:41.108 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:10:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:41.110760 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data
15:10:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:41.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/logs
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.188:32887
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.188
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37067
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.188
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:41.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:10:41.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:41.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.409714 27226 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:41.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.410333 27226 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:41.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.411077 27226 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:41.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.442086 27226 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:41.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.442349 27226 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:41.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.442787 27226 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:41.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.443095 27226 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:41.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.443364 27226 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:41.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.443656 27226 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:41.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.443919 27226 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:41.482636 27226 master_runner.cc:386] Master server non-default flags:
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.188
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.188:32887
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.188
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37067
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/logs
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:41.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:41.484206 27226 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:41.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:41.485255 27226 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:41.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.496215 27232 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:43.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:42.903146 27231 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 27226
15:10:43.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.215691 27231 kernel_stack_watchdog.cc:198] Thread 27226 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 402ms:
15:10:43.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:10:43.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:10:43.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:10:43.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:10:43.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:41.496745 27233 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:43.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.216764 27226 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.720s user 0.517s sys 1.194s
15:10:43.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.217131 27226 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.721s user 0.517s sys 1.194s
15:10:43.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.219173 27235 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.222963 27234 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1725 milliseconds
15:10:43.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.222975 27226 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.224232 27226 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:43.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.226605 27226 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:43.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.228008 27226 hybrid_clock.cc:648] HybridClock initialized: now 1765638643227978 us; error 32 us; skew 500 ppm
15:10:43.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.228873 27226 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:43 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638643, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:43.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.235055 27226 init.cc:377] Logged in from keytab as kudu/127.26.64.188@KRBTEST.COM (short username kudu)
15:10:43.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.239718 27226 webserver.cc:492] Webserver started at http://127.26.64.188:37067/ using document root <none> and password file <none>
15:10:43.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.240696 27226 fs_manager.cc:362] Metadata directory not provided
15:10:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.240963 27226 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:43.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.248968 27226 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
15:10:43.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.253279 27244 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:43.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.254299 27226 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
15:10:43.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.254622 27226 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:43.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65b363a0f05e4d7fa201cbc9853bb322"
15:10:43.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:39 on dist-test-slave-xht5"
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.256455 27226 fs_report.cc:389] FS layout report
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/data
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:43.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:43.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.318152 27226 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.328972 27226 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:43.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.329392 27226 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:43.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.406793 27226 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.188:32887
15:10:43.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.406844 27295 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.188:32887 every 8 connection(s)
15:10:43.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.410724 27226 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:10:43.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.416021 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 27226
15:10:43.416 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:43.416 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.190:36655
15:10:43.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:10:43.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.418496 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:43.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:43.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:43.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/logs
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.190:36655
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.190
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44727
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.190
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:43.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:10:43.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:43.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.425163 27296 sys_catalog.cc:263] Verifying existing consensus state
15:10:43.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.430300 27296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap starting.
15:10:43.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.468834 27296 log.cc:826] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Log is configured to *not* fsync() on all Append() calls
15:10:43.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.488476 27296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:10:43.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.489456 27296 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap complete.
15:10:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.514544 27296 raft_consensus.cc:359] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:43.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.516680 27296 raft_consensus.cc:740] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 65b363a0f05e4d7fa201cbc9853bb322, State: Initialized, Role: FOLLOWER
15:10:43.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.517378 27296 consensus_queue.cc:260] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.520277 27300 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:43.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.521106 27300 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: FOLLOWER
15:10:43.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.522076 27296 sys_catalog.cc:565] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: configured and running, proceeding with master startup.
15:10:43.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.555356 27311 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:10:43.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.555653 27311 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: loading cluster ID for follower catalog manager: success
15:10:43.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.562875 27311 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: acquiring CA information for follower catalog manager: success
15:10:43.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.568089 27311 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:10:43.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.549211 27226 master_runner.cc:186] Error getting master registration for 127.26.64.190:36655: Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111),
15:10:43.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.574051 27226 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.784724 27298 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:43.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.785331 27298 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:43.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.785961 27298 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:43.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.817677 27298 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:43.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.818014 27298 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:43.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.818531 27298 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:43.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.818868 27298 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:43.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.819157 27298 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:43.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.819433 27298 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:43.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.819638 27298 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:10:43.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.857398 27298 master_runner.cc:386] Master server non-default flags:
15:10:43.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:43.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:43.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:43.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:43.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data
15:10:43.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.190
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.190:36655
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.190
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44727
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/logs
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:43.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:43.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.859022 27298 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:43.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:43.860174 27298 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:43.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.871348 27319 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:44.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:44.845088 27325 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:44.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:44.845736 27325 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:44.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:44.868707 27246 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111)
15:10:44.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:44.884119 27325 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:10:44.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:44.885210 27246 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655): Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111)
15:10:44.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:44.895466 27246 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:44.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:44.896096 27246 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322; no voters: 86daac6df1334b1bb65f5108dcdd44cc, b6eade2b7fd24c10aae109da5e2e6718
15:10:44.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:44.899523 27325 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
15:10:46.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:45.274875 27318 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 27298
15:10:46.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.132623 27318 kernel_stack_watchdog.cc:198] Thread 27298 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
15:10:46.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:10:46.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:10:46.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:10:46.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:10:46.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:43.872706 27320 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:46.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.135049 27298 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.260s user 0.000s sys 0.001s
15:10:46.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.135443 27298 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.261s user 0.000s sys 0.002s
15:10:46.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.136801 27321 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2263 milliseconds
15:10:46.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.140051 27298 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:46.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.142064 27323 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:46.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.143698 27298 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:46.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.146556 27298 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:46.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.147971 27298 hybrid_clock.cc:648] HybridClock initialized: now 1765638646147905 us; error 64 us; skew 500 ppm
15:10:46.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.148871 27298 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:46.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:46 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638646, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:46.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.154834 27298 init.cc:377] Logged in from keytab as kudu/127.26.64.190@KRBTEST.COM (short username kudu)
15:10:46.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.160967 27298 webserver.cc:492] Webserver started at http://127.26.64.190:44727/ using document root <none> and password file <none>
15:10:46.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.161914 27298 fs_manager.cc:362] Metadata directory not provided
15:10:46.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.162149 27298 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:46.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.170641 27298 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
15:10:46.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.176129 27335 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:46.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.177294 27298 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
15:10:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.177654 27298 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86daac6df1334b1bb65f5108dcdd44cc"
15:10:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:35 on dist-test-slave-xht5"
15:10:46.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.180342 27298 fs_report.cc:389] FS layout report
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/data
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:46.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:46.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.244573 27298 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.250090 27298 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:46.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.250582 27298 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:46.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.341945 27298 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.190:36655
15:10:46.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.342007 27386 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.190:36655 every 8 connection(s)
15:10:46.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.345094 27298 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:10:46.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.345603 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 27298
15:10:46.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:46.346 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.189:37765
15:10:46.346 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:10:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.348556 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:46.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/logs
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:46.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.189:37765
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.189
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45529
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.189
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:10:46.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:46.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.360306 27387 sys_catalog.cc:263] Verifying existing consensus state
15:10:46.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.367923 27387 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap starting.
15:10:46.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.426733 27387 log.cc:826] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Log is configured to *not* fsync() on all Append() calls
15:10:46.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.448530 27387 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:10:46.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.449517 27387 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap complete.
15:10:46.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.484283 27387 raft_consensus.cc:359] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:46.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.488314 27387 raft_consensus.cc:740] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86daac6df1334b1bb65f5108dcdd44cc, State: Initialized, Role: FOLLOWER
15:10:46.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.489367 27387 consensus_queue.cc:260] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:46.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.505911 27387 sys_catalog.cc:565] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: configured and running, proceeding with master startup.
15:10:46.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.503578 27392 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:46.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.515245 27392 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:10:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.603330 27403 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:10:46.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.603600 27403 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: loading cluster ID for follower catalog manager: success
15:10:46.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.615271 27403 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: acquiring CA information for follower catalog manager: success
15:10:46.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:46 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638646, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:10:46.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.628149 27403 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:10:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.648572 27298 master_runner.cc:186] Error getting master registration for 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111),
15:10:46.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.675384 27298 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.807636 27389 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.808305 27389 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.809293 27389 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:46.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.849534 27389 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:46.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.849789 27389 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:46.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.850163 27389 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:46.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.850481 27389 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:46.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.850756 27389 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:46.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.851025 27389 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:46.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.851234 27389 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:10:46.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.907369 27389 master_runner.cc:386] Master server non-default flags:
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.189
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.189:37765
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.189
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45529
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/logs
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:46.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:46.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.909119 27389 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:46.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:46.910573 27389 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:46.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.924158 27412 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:47.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.107427 27418 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:47.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.108932 27418 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:47.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.124737 27418 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:10:47.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:47.162956 27246 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:47.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.188093 27362 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "65b363a0f05e4d7fa201cbc9853bb322" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "86daac6df1334b1bb65f5108dcdd44cc" is_pre_election: true
15:10:47.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.189323 27362 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 65b363a0f05e4d7fa201cbc9853bb322 in term 1.
15:10:47.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.191046 27246 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322, 86daac6df1334b1bb65f5108dcdd44cc; no voters: b6eade2b7fd24c10aae109da5e2e6718
15:10:47.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.192169 27418 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Leader pre-election won for term 2
15:10:47.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.192631 27418 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:10:47.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.193020 27418 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 1 FOLLOWER]: Advancing to term 2
15:10:47.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.206215 27418 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:47.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.227473 27362 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "65b363a0f05e4d7fa201cbc9853bb322" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "86daac6df1334b1bb65f5108dcdd44cc"
15:10:47.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.228264 27362 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 1 FOLLOWER]: Advancing to term 2
15:10:47.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:47.235234 27246 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:47.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.236552 27418 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 2 election: Requested vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:10:47.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.252862 27362 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 65b363a0f05e4d7fa201cbc9853bb322 in term 2.
15:10:47.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.254634 27246 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322, 86daac6df1334b1bb65f5108dcdd44cc; no voters: b6eade2b7fd24c10aae109da5e2e6718
15:10:47.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.255789 27418 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Leader election won for term 2
15:10:47.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.273948 27418 raft_consensus.cc:697] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 LEADER]: Becoming Leader. State: Replica: 65b363a0f05e4d7fa201cbc9853bb322, State: Running, Role: LEADER
15:10:47.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.275573 27418 consensus_queue.cc:237] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:47.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.306766 27422 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 65b363a0f05e4d7fa201cbc9853bb322. Latest consensus state: current_term: 2 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:47.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.309141 27422 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: LEADER
15:10:47.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.313047 27424 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:10:47.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.330938 27424 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:10:47.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.337828 27424 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:10:47.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.338115 27424 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:10:47.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.349833 27424 catalog_manager.cc:1514] Loading token signing keys...
15:10:47.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.358450 27424 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Loaded TSK: 0
15:10:47.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.359717 27424 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:10:47.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.810959 27362 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 2 FOLLOWER]: Refusing update from remote peer 65b363a0f05e4d7fa201cbc9853bb322: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
15:10:47.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.813077 27422 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [LEADER]: Connected to new peer: Peer: permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
15:10:47.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:47.855893 27246 consensus_peers.cc:597] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 -> Peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Couldn't send request to peer b6eade2b7fd24c10aae109da5e2e6718. Status: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:10:47.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.869819 27427 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 65b363a0f05e4d7fa201cbc9853bb322. Latest consensus state: current_term: 2 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:47.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.875489 27427 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:10:47.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.891158 27427 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:47.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.892230 27427 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:10:47.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.908061 27422 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:47.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:47.909070 27422 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: LEADER
15:10:48.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:48.596410 27226 master_runner.cc:186] Error getting master registration for 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111),
15:10:48.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:48.597088 27226 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:49.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:48.327806 27411 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 27389
15:10:49.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:49.099078 27411 kernel_stack_watchdog.cc:198] Thread 27389 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
15:10:49.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:10:49.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:10:49.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:10:49.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:10:49.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:46.925294 27413 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:49.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:49.101755 27415 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:49.103646 27389 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.179s user 0.739s sys 1.088s
15:10:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:49.103978 27389 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.179s user 0.739s sys 1.088s
15:10:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:49.105357 27414 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2179 milliseconds
15:10:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.105429 27389 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:49.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.106734 27389 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:49.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.109705 27389 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:49.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.111111 27389 hybrid_clock.cc:648] HybridClock initialized: now 1765638649111010 us; error 111 us; skew 500 ppm
15:10:49.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.111922 27389 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:49.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:49 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638649, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:49.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.118062 27389 init.cc:377] Logged in from keytab as kudu/127.26.64.189@KRBTEST.COM (short username kudu)
15:10:49.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.124440 27389 webserver.cc:492] Webserver started at http://127.26.64.189:45529/ using document root <none> and password file <none>
15:10:49.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.125324 27389 fs_manager.cc:362] Metadata directory not provided
15:10:49.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.125614 27389 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:49.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.134536 27389 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
15:10:49.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.139298 27441 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.140437 27389 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
15:10:49.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.140759 27389 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:49.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6eade2b7fd24c10aae109da5e2e6718"
15:10:49.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:37 on dist-test-slave-xht5"
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.142706 27389 fs_report.cc:389] FS layout report
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/data
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:49.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:49.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.191197 27389 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:49.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.196969 27389 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:49.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.197405 27389 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:49.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.285171 27389 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.189:37765
15:10:49.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.285316 27492 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.189:37765 every 8 connection(s)
15:10:49.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.288008 27389 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:10:49.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.289811 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 27389
15:10:49.290 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:49.291 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.64.188:32887(127.26.64.188:32887)] connecting to peer
15:10:49.293 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.64.190:36655(127.26.64.190:36655)] connecting to peer
15:10:49.299 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.64.189:37765(127.26.64.189:37765)] connecting to peer
15:10:49.303 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.26.64.188:32887(127.26.64.188:32887)] Successfully connected to peer
15:10:49.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.303437 27493 sys_catalog.cc:263] Verifying existing consensus state
15:10:49.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.310240 27493 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Bootstrap starting.
15:10:49.315 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.26.64.189:37765(127.26.64.189:37765)] Successfully connected to peer
15:10:49.325 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
15:10:49.325 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc
15:10:49.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:49 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:49.333 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:10:49.334 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
15:10:49.335 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.26.64.189
15:10:49.340 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
15:10:49.340 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc
15:10:49.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:49 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:49.345 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.26.64.190:36655(127.26.64.190:36655)] Successfully connected to peer
15:10:49.354 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:10:49.354 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
15:10:49.358 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
15:10:49.359 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc
15:10:49.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:49 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:49.364 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:10:49.365 [INFO - kudu-nio-1] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
15:10:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.369016 27493 log.cc:826] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Log is configured to *not* fsync() on all Append() calls
15:10:49.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.389372 27493 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:10:49.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.390393 27493 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Bootstrap complete.
15:10:49.394 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.26.64.190
15:10:49.406 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.26.64.188
15:10:49.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.438947 27493 raft_consensus.cc:359] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:49.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.442451 27493 raft_consensus.cc:740] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b6eade2b7fd24c10aae109da5e2e6718, State: Initialized, Role: FOLLOWER
15:10:49.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.443413 27493 consensus_queue.cc:260] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:49.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:49 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:10:49.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:49 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.64.188@KRBTEST.COM
15:10:49.497 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0xc03061cb, L:/127.0.0.1:39682 - R:/127.26.64.190:36655] using SASL/GSSAPI
15:10:49.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.503129 27493 sys_catalog.cc:565] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: configured and running, proceeding with master startup.
15:10:49.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.499804 27502 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:49.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.505380 27502 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:10:49.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:49 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.64.189@KRBTEST.COM
15:10:49.554 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x6f3317e2, L:/127.0.0.1:53432 - R:/127.26.64.188:32887] using SASL/GSSAPI
15:10:49.569 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x577c5a85, L:/127.0.0.1:54444 - R:/127.26.64.189:37765] using SASL/GSSAPI
15:10:49.577 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
15:10:49.578 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=287), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.64.188:32887, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.64.190:36655, [8ms] Sub RPC ConnectToMaster: sending RPC to server master-127.26.64.189:37765, [210ms] Sub RPC ConnectToMaster: received response from server master-127.26.64.190:36655: OK, [283ms] Sub RPC ConnectToMaster: received response from server master-127.26.64.188:32887: OK, deferred=null) after lookup
15:10:49.597 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.26.64.189:37765: Catalog manager is not initialized. State: Starting
15:10:49.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:49 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638643, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.188@KRBTEST.COM for kudu/127.26.64.189@KRBTEST.COM
15:10:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.687567 27468 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 1 FOLLOWER]: Advancing to term 2
15:10:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.724459 27513 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:10:49.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.724781 27513 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: loading cluster ID for follower catalog manager: success
15:10:49.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.728444 27502 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 65b363a0f05e4d7fa201cbc9853bb322. Latest consensus state: current_term: 2 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:49.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.729135 27502 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:10:49.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.731742 27513 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: acquiring CA information for follower catalog manager: success
15:10:49.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.734244 27518 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:49.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.735000 27518 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:10:49.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:49.736992 27513 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:10:49.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:49 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638649, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:10:50.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:50.267830 27422 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:50.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:50.269925 27422 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: LEADER
15:10:50.598 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.188:32887
15:10:50.598 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:10:50.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:50.599068 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 27226
15:10:50.629 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.26.64.188:32887(127.26.64.188:32887)] handling channelInactive
15:10:50.630 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:50.630 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.190:36655
15:10:50.630 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.64.188:32887(127.26.64.188:32887)] cleaning up while in state READY due to: connection closed
15:10:50.630 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:10:50.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:50.630833 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 27298
15:10:50.661 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.26.64.190:36655(127.26.64.190:36655)] handling channelInactive
15:10:50.661 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:50.661 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.189:37765
15:10:50.661 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.64.190:36655(127.26.64.190:36655)] cleaning up while in state READY due to: connection closed
15:10:50.661 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:10:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:50.662211 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 27389
15:10:50.686 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.26.64.189:37765(127.26.64.189:37765)] handling channelInactive
15:10:50.687 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:50.687 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.64.189:37765(127.26.64.189:37765)] cleaning up while in state READY due to: connection closed
15:10:50.687 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.188:32887
15:10:50.687 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:10:50.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:50.688437 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:50.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:50.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:50.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data
15:10:50.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:50.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/logs
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.188:32887
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.188
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37067
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.188
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:10:50.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:50.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:50.995674 27523 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:50.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:50.996243 27523 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:50.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:50.996869 27523 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:51.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:51.030300 27523 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:51.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:51.030591 27523 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:51.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:51.030961 27523 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:51.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:51.031176 27523 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:51.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:51.031401 27523 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:51.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:51.031594 27523 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:51.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:51.031739 27523 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:51.070901 27523 master_runner.cc:386] Master server non-default flags:
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.188
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.188:32887
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.188
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37067
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/logs
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:51.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:51.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:51.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:51.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:51.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:51.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:51.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:51.072172 27523 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:51.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:51.073223 27523 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:51.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:51.084828 27530 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:52.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:52.488085 27529 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 27523
15:10:52.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:52.907898 27523 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.823s user 0.692s sys 1.126s
15:10:52.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:52.908293 27523 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.823s user 0.693s sys 1.126s
15:10:52.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:51.086038 27531 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:52.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:52.909641 27532 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1822 milliseconds
15:10:52.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:52.910939 27533 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:52.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.910887 27523 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:52.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.913854 27523 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:52.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.916258 27523 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:52.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.917618 27523 hybrid_clock.cc:648] HybridClock initialized: now 1765638652917577 us; error 43 us; skew 500 ppm
15:10:52.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.918499 27523 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:52.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:52 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638652, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:52.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.924582 27523 init.cc:377] Logged in from keytab as kudu/127.26.64.188@KRBTEST.COM (short username kudu)
15:10:52.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.929714 27523 webserver.cc:492] Webserver started at http://127.26.64.188:37067/ using document root <none> and password file <none>
15:10:52.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.930683 27523 fs_manager.cc:362] Metadata directory not provided
15:10:52.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.930898 27523 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:52.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.938782 27523 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.006s sys 0.002s
15:10:52.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.943339 27542 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:52.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.944633 27523 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.945081 27523 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65b363a0f05e4d7fa201cbc9853bb322"
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:39 on dist-test-slave-xht5"
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:52.947113 27523 fs_report.cc:389] FS layout report
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/data
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:52.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:53.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.025146 27523 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:53.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.030835 27523 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:53.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.031253 27523 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:53.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.107116 27523 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.188:32887
15:10:53.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.107234 27593 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.188:32887 every 8 connection(s)
15:10:53.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.110000 27523 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:10:53.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.115100 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 27523
15:10:53.115 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:53.116 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.190:36655
15:10:53.116 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.117802 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/logs
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:53.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.190:36655
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.190
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44727
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.190
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:10:53.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:53.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.125346 27594 sys_catalog.cc:263] Verifying existing consensus state
15:10:53.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.130591 27594 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap starting.
15:10:53.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.168964 27594 log.cc:826] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Log is configured to *not* fsync() on all Append() calls
15:10:53.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.186194 27594 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:10:53.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.186976 27594 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap complete.
15:10:53.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.209234 27594 raft_consensus.cc:359] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:53.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.211606 27594 raft_consensus.cc:740] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 65b363a0f05e4d7fa201cbc9853bb322, State: Initialized, Role: FOLLOWER
15:10:53.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.212466 27594 consensus_queue.cc:260] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:53.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.215272 27598 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:53.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.216061 27598 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: FOLLOWER
15:10:53.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.217149 27594 sys_catalog.cc:565] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: configured and running, proceeding with master startup.
15:10:53.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.248997 27609 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:10:53.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.249681 27609 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: loading cluster ID for follower catalog manager: success
15:10:53.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.257431 27609 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: acquiring CA information for follower catalog manager: success
15:10:53.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.263182 27609 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:10:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.243841 27523 master_runner.cc:186] Error getting master registration for 127.26.64.190:36655: Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111),
15:10:53.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.270617 27523 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:53.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.456992 27596 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:53.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.457674 27596 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:53.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.458381 27596 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:53.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.493027 27596 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:53.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.493328 27596 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:53.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.493731 27596 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:53.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.494031 27596 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:53.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.494323 27596 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:53.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.494648 27596 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:53.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.494869 27596 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:10:53.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.541584 27596 master_runner.cc:386] Master server non-default flags:
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.190
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.190:36655
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.190
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44727
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/logs
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:53.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:53.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:53.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:53.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:53.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:53.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:53.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:53.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:53.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:53.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.543169 27596 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:53.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:53.544241 27596 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:53.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.555390 27617 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:55.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.090549 27623 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:55.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.092993 27623 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:55.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.129794 27623 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:10:55.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:55.135728 27544 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111)
15:10:55.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:55.168444 27544 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655): Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111)
15:10:55.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:55.169320 27544 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:55.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.169807 27544 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322; no voters: 86daac6df1334b1bb65f5108dcdd44cc, b6eade2b7fd24c10aae109da5e2e6718
15:10:55.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.170799 27623 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
15:10:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:54.959383 27616 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 27596
15:10:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:55.430089 27616 kernel_stack_watchdog.cc:198] Thread 27596 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
15:10:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:10:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:10:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:10:55.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:10:55.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:53.557109 27618 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:55.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:55.431836 27596 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.875s user 0.000s sys 0.003s
15:10:55.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:55.432204 27596 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.875s user 0.000s sys 0.003s
15:10:55.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:55.436162 27619 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1873 milliseconds
15:10:55.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:55.437729 27620 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:55.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.437832 27596 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:55.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.439311 27596 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:55.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.442287 27596 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:55.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.443753 27596 hybrid_clock.cc:648] HybridClock initialized: now 1765638655443668 us; error 82 us; skew 500 ppm
15:10:55.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.444908 27596 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:55.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:55 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638655, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:55.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.452241 27596 init.cc:377] Logged in from keytab as kudu/127.26.64.190@KRBTEST.COM (short username kudu)
15:10:55.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.458113 27596 webserver.cc:492] Webserver started at http://127.26.64.190:44727/ using document root <none> and password file <none>
15:10:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.459095 27596 fs_manager.cc:362] Metadata directory not provided
15:10:55.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.459309 27596 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:55.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.468155 27596 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
15:10:55.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.473186 27633 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:55.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.474303 27596 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
15:10:55.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.474663 27596 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86daac6df1334b1bb65f5108dcdd44cc"
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:35 on dist-test-slave-xht5"
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.476580 27596 fs_report.cc:389] FS layout report
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/data
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:55.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:55.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.533646 27596 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.543004 27596 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.543503 27596 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.648568 27596 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.190:36655
15:10:55.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.648720 27684 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.190:36655 every 8 connection(s)
15:10:55.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.651432 27596 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:10:55.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.659277 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 27596
15:10:55.659 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:55.659 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.189:37765
15:10:55.660 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:10:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.661356 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data
15:10:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/logs
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.189:37765
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.189
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45529
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.189
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:55.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:10:55.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:55.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.666602 27685 sys_catalog.cc:263] Verifying existing consensus state
15:10:55.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.674943 27685 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap starting.
15:10:55.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.731971 27685 log.cc:826] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Log is configured to *not* fsync() on all Append() calls
15:10:55.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.753813 27685 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:10:55.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.754884 27685 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap complete.
15:10:55.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.792649 27685 raft_consensus.cc:359] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:55.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.797137 27685 raft_consensus.cc:740] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86daac6df1334b1bb65f5108dcdd44cc, State: Initialized, Role: FOLLOWER
15:10:55.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.798334 27685 consensus_queue.cc:260] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.805109 27690 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.806185 27690 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:10:55.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.806347 27685 sys_catalog.cc:565] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: configured and running, proceeding with master startup.
15:10:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.863813 27701 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:10:55.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.864319 27701 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: loading cluster ID for follower catalog manager: success
15:10:55.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.872592 27701 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: acquiring CA information for follower catalog manager: success
15:10:55.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.878854 27701 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:10:55.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:55 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638655, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:10:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.906061 27596 master_runner.cc:186] Error getting master registration for 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111),
15:10:55.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:55.931187 27596 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:56.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.097427 27687 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:56.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.098212 27687 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:56.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.099370 27687 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:56.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.158476 27687 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:56.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.158821 27687 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:56.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.159304 27687 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:56.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.159660 27687 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.159988 27687 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.160319 27687 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.160578 27687 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.217828 27687 master_runner.cc:386] Master server non-default flags:
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.189
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:56.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.189:37765
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.189
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45529
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/logs
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.219556 27687 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:56.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.220871 27687 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:56.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.245563 27710 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:56.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.729848 27717 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:10:56.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.730657 27717 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:56.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.745569 27717 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:10:56.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.770740 27544 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:56.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.774385 27660 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "65b363a0f05e4d7fa201cbc9853bb322" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "86daac6df1334b1bb65f5108dcdd44cc" is_pre_election: true
15:10:56.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.775601 27660 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 65b363a0f05e4d7fa201cbc9853bb322 in term 2.
15:10:56.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.777420 27544 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322, 86daac6df1334b1bb65f5108dcdd44cc; no voters: b6eade2b7fd24c10aae109da5e2e6718
15:10:56.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.778481 27717 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Leader pre-election won for term 3
15:10:56.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.778885 27717 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:10:56.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.779284 27717 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 2 FOLLOWER]: Advancing to term 3
15:10:56.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.793237 27717 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:56.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.797376 27660 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "65b363a0f05e4d7fa201cbc9853bb322" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "86daac6df1334b1bb65f5108dcdd44cc"
15:10:56.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.798135 27660 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 2 FOLLOWER]: Advancing to term 3
15:10:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.809736 27717 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 3 election: Requested vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:10:56.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.813946 27544 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:10:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.834569 27660 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 65b363a0f05e4d7fa201cbc9853bb322 in term 3.
15:10:56.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.836122 27544 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322, 86daac6df1334b1bb65f5108dcdd44cc; no voters: b6eade2b7fd24c10aae109da5e2e6718
15:10:56.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.837536 27717 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Leader election won for term 3
15:10:56.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.851419 27717 raft_consensus.cc:697] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 LEADER]: Becoming Leader. State: Replica: 65b363a0f05e4d7fa201cbc9853bb322, State: Running, Role: LEADER
15:10:56.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.853052 27717 consensus_queue.cc:237] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:56.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.872404 27722 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 65b363a0f05e4d7fa201cbc9853bb322. Latest consensus state: current_term: 3 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:56.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.873800 27722 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: LEADER
15:10:56.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.883780 27724 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:10:56.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.894732 27724 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:10:56.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.900230 27724 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:10:56.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.900792 27724 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:10:56.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.907150 27724 catalog_manager.cc:1514] Loading token signing keys...
15:10:56.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.912895 27724 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Loaded TSK: 0
15:10:56.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:56.914749 27724 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:10:57.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.262533 27660 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 3 FOLLOWER]: Refusing update from remote peer 65b363a0f05e4d7fa201cbc9853bb322: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
15:10:57.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.265785 27722 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [LEADER]: Connected to new peer: Peer: permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
15:10:57.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:57.279603 27544 consensus_peers.cc:597] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 -> Peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Couldn't send request to peer b6eade2b7fd24c10aae109da5e2e6718. Status: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:10:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.328210 27725 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 65b363a0f05e4d7fa201cbc9853bb322. Latest consensus state: current_term: 3 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:57.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.329294 27725 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:10:57.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.330847 27722 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:57.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.331570 27722 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: LEADER
15:10:57.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.336757 27725 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:57.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.338675 27725 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:10:57.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.251531 27713 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:57.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:56.246732 27711 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:10:57.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:57.773298 27712 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1521 milliseconds
15:10:57.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.773430 27687 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:10:57.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.774971 27687 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:10:57.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.778012 27687 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:10:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.779527 27687 hybrid_clock.cc:648] HybridClock initialized: now 1765638657779419 us; error 106 us; skew 500 ppm
15:10:57.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.780656 27687 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:10:57.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:57 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638657, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:57.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.787776 27687 init.cc:377] Logged in from keytab as kudu/127.26.64.189@KRBTEST.COM (short username kudu)
15:10:57.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.793880 27687 webserver.cc:492] Webserver started at http://127.26.64.189:45529/ using document root <none> and password file <none>
15:10:57.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.794803 27687 fs_manager.cc:362] Metadata directory not provided
15:10:57.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.794987 27687 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:10:57.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.803537 27687 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.001s
15:10:57.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.809958 27735 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:10:57.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.811126 27687 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
15:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.811470 27687 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6eade2b7fd24c10aae109da5e2e6718"
15:10:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:37 on dist-test-slave-xht5"
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.813484 27687 fs_report.cc:389] FS layout report
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/data
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:10:57.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:10:57.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.866885 27687 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:10:57.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.872958 27687 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:10:57.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.873447 27687 kserver.cc:163] Server-wide thread pool size limit: 3276
15:10:57.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.962545 27687 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.189:37765
15:10:57.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.962644 27786 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.189:37765 every 8 connection(s)
15:10:57.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.965559 27687 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:10:57.968 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:57.968 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.64.188:32887(127.26.64.188:32887)] connecting to peer
15:10:57.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.967504 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 27687
15:10:57.970 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.26.64.188:32887(127.26.64.188:32887)] Successfully connected to peer
15:10:57.975 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
15:10:57.976 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc
15:10:57.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:57 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:57.980 [DEBUG - kudu-nio-1] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:10:57.980 [INFO - kudu-nio-1] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
15:10:57.981 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.26.64.188
15:10:57.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.984817 27787 sys_catalog.cc:263] Verifying existing consensus state
15:10:57.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:57.993657 27787 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Bootstrap starting.
15:10:57.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:57 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638639, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.64.188@KRBTEST.COM
15:10:58.015 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x11171d59, L:/127.0.0.1:55622 - R:/127.26.64.188:32887] using SASL/GSSAPI
15:10:58.028 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
15:10:58.029 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
15:10:58.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.058043 27787 log.cc:826] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Log is configured to *not* fsync() on all Append() calls
15:10:58.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:58 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638658, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:10:58.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
15:10:58.070 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:58.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.078330 27787 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:10:58.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.079267 27787 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Bootstrap complete.
15:10:58.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.104104 27787 raft_consensus.cc:359] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:58.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.107288 27787 raft_consensus.cc:740] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: b6eade2b7fd24c10aae109da5e2e6718, State: Initialized, Role: FOLLOWER
15:10:58.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.108296 27787 consensus_queue.cc:260] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:10:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.124845 27793 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:58.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.127426 27793 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:10:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.127761 27787 sys_catalog.cc:565] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: configured and running, proceeding with master startup.
15:10:58.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.181675 27804 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:10:58.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.182546 27804 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: loading cluster ID for follower catalog manager: success
15:10:58.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.193132 27804 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: acquiring CA information for follower catalog manager: success
15:10:58.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.198330 27804 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:10:58.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:10:58 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638657, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:10:58.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.295468 27523 master_runner.cc:232] Existing masters have differing terms: 3 vs 2
15:10:58.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.295955 27523 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:10:58.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.312737 27762 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 2 FOLLOWER]: Advancing to term 3
15:10:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.334640 27813 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:58.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.335708 27813 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:10:58.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.339570 27793 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 65b363a0f05e4d7fa201cbc9853bb322. Latest consensus state: current_term: 3 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:58.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.340245 27793 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:10:58.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.828481 27814 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:10:58.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:58.829216 27814 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: LEADER
15:10:59.070 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.188:32887
15:10:59.071 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:10:59.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:59.072260 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 27523
15:10:59.102 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.26.64.188:32887(127.26.64.188:32887)] handling channelInactive
15:10:59.102 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:59.102 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.64.188:32887(127.26.64.188:32887)] cleaning up while in state READY due to: connection closed
15:10:59.102 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.190:36655
15:10:59.102 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:10:59.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:59.102823 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 27596
15:10:59.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:59.143 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.189:37765
15:10:59.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:10:59.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:59.143553 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 27687
15:10:59.170 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:10:59.170 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.188:32887
15:10:59.171 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:59.172974 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/logs
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.188:32887
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.188
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37067
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.188
15:10:59.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:10:59.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:10:59.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.501981 27818 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:10:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.502719 27818 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:10:59.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.503585 27818 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:10:59.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.537016 27818 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:10:59.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.537297 27818 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:10:59.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.537726 27818 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:10:59.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.538041 27818 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:10:59.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.538331 27818 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:10:59.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.538643 27818 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:10:59.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.538864 27818 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:59.577018 27818 master_runner.cc:386] Master server non-default flags:
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.188
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.188:32887
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:10:59.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.188
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37067
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/logs
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:10:59.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:59.578680 27818 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:10:59.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:10:59.579741 27818 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:10:59.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.591763 27824 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:00.994489 27823 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 27818
15:11:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.092186 27823 kernel_stack_watchdog.cc:198] Thread 27818 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
15:11:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:11:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:11:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:11:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:11:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.095118 27826 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1500 milliseconds
15:11:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.095085 27818 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.502s user 0.000s sys 0.001s
15:11:01.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:10:59.592237 27825 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:01.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.095813 27818 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.503s user 0.000s sys 0.002s
15:11:01.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.097067 27827 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:01.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.097404 27818 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:11:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.101272 27818 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:11:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.103899 27818 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:11:01.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.105243 27818 hybrid_clock.cc:648] HybridClock initialized: now 1765638661105263 us; error 88 us; skew 500 ppm
15:11:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.106182 27818 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:11:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:01 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638661, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:11:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.112640 27818 init.cc:377] Logged in from keytab as kudu/127.26.64.188@KRBTEST.COM (short username kudu)
15:11:01.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.118216 27818 webserver.cc:492] Webserver started at http://127.26.64.188:37067/ using document root <none> and password file <none>
15:11:01.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.119415 27818 fs_manager.cc:362] Metadata directory not provided
15:11:01.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.119676 27818 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:11:01.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.129755 27818 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.000s
15:11:01.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.134879 27838 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:11:01.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.136178 27818 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
15:11:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.136523 27818 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:11:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65b363a0f05e4d7fa201cbc9853bb322"
15:11:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:39 on dist-test-slave-xht5"
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.138656 27818 fs_report.cc:389] FS layout report
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/data
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:11:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:11:01.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.193068 27818 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:11:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.205518 27818 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:11:01.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.206038 27818 kserver.cc:163] Server-wide thread pool size limit: 3276
15:11:01.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.276607 27818 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.188:32887
15:11:01.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.276700 27889 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.188:32887 every 8 connection(s)
15:11:01.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.279458 27818 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:11:01.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.287142 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 27818
15:11:01.287 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:11:01.288 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.190:36655
15:11:01.288 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:11:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.289350 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:11:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:11:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:11:01.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data
15:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/logs
15:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:11:01.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.190:36655
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.190
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44727
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.190
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:11:01.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.290916 27890 sys_catalog.cc:263] Verifying existing consensus state
15:11:01.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.298970 27890 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap starting.
15:11:01.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.352451 27890 log.cc:826] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Log is configured to *not* fsync() on all Append() calls
15:11:01.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.374150 27890 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:11:01.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.375216 27890 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap complete.
15:11:01.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.399066 27890 raft_consensus.cc:359] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:01.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.402007 27890 raft_consensus.cc:740] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 65b363a0f05e4d7fa201cbc9853bb322, State: Initialized, Role: FOLLOWER
15:11:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.403004 27890 consensus_queue.cc:260] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:01.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.411355 27894 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.413303 27890 sys_catalog.cc:565] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: configured and running, proceeding with master startup.
15:11:01.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.413116 27894 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: FOLLOWER
15:11:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.445168 27905 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:11:01.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.445706 27905 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: loading cluster ID for follower catalog manager: success
15:11:01.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.453282 27905 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: acquiring CA information for follower catalog manager: success
15:11:01.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.458462 27905 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:11:01.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.447631 27818 master_runner.cc:186] Error getting master registration for 127.26.64.190:36655: Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111),
15:11:01.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.469635 27818 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:11:01.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.733412 27892 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:11:01.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.734226 27892 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:11:01.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.735394 27892 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:11:01.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.778043 27892 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:11:01.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.778272 27892 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:11:01.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.778575 27892 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:11:01.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.778828 27892 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:11:01.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.779013 27892 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:11:01.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.779194 27892 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:11:01.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.779326 27892 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:11:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.815490 27892 master_runner.cc:386] Master server non-default flags:
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.190
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.190:36655
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.190
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44727
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/logs
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:11:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:11:01.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:11:01.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:11:01.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:11:01.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:11:01.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:11:01.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:11:01.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:11:01.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.816803 27892 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:11:01.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:01.817826 27892 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:11:01.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.829285 27913 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:01.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.864105 26587 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8bc0 after lost signal to thread 26457
15:11:01.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.865798 26587 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9960 after lost signal to thread 26590
15:11:01.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.872871 26867 debug-util.cc:398] Leaking SignalData structure 0x7b08000baba0 after lost signal to thread 26742
15:11:01.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.873998 26867 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad420 after lost signal to thread 26870
15:11:02.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:02.682873 27919 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:11:02.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:02.683745 27919 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:02.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:02.722112 27840 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111)
15:11:02.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:02.726646 27919 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:11:02.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:02.755244 27840 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655): Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111)
15:11:02.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:02.763702 27840 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:11:02.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:02.764185 27840 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322; no voters: 86daac6df1334b1bb65f5108dcdd44cc, b6eade2b7fd24c10aae109da5e2e6718
15:11:02.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:02.765625 27919 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:11:04.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:03.232292 27912 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 27892
15:11:04.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:04.466768 27912 kernel_stack_watchdog.cc:198] Thread 27892 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 398ms:
15:11:04.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
15:11:04.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
15:11:04.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
15:11:04.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
15:11:04.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:01.830498 27914 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:04.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:04.477699 27915 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2635 milliseconds
15:11:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:04.479624 27892 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.648s user 0.000s sys 0.007s
15:11:04.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:04.479972 27892 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.649s user 0.000s sys 0.007s
15:11:04.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:04.483261 27916 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:04.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.483276 27892 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:11:04.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.484987 27892 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:11:04.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.498610 27892 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:11:04.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.502247 27892 hybrid_clock.cc:648] HybridClock initialized: now 1765638664501744 us; error 496 us; skew 500 ppm
15:11:04.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.505023 27892 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:11:04.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:04 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638664, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:11:04.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.517936 27892 init.cc:377] Logged in from keytab as kudu/127.26.64.190@KRBTEST.COM (short username kudu)
15:11:04.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.555930 27892 webserver.cc:492] Webserver started at http://127.26.64.190:44727/ using document root <none> and password file <none>
15:11:04.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.557298 27892 fs_manager.cc:362] Metadata directory not provided
15:11:04.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.557597 27892 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:11:04.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.580714 27892 fs_manager.cc:714] Time spent opening directory manager: real 0.018s user 0.005s sys 0.004s
15:11:04.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.592331 27929 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:11:04.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.593731 27892 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.005s sys 0.004s
15:11:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.594173 27892 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:11:04.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86daac6df1334b1bb65f5108dcdd44cc"
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:35 on dist-test-slave-xht5"
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.596899 27892 fs_report.cc:389] FS layout report
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/data
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:11:04.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:11:04.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:11:04.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.691592 27892 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:11:04.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.700471 27892 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:11:04.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:04.701366 27892 kserver.cc:163] Server-wide thread pool size limit: 3276
15:11:05.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.004763 27967 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:11:05.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.005722 27967 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:05.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.019065 27967 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:11:05.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.040715 27840 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655): Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111)
15:11:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.051156 27840 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:11:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.051900 27840 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322; no voters: 86daac6df1334b1bb65f5108dcdd44cc, b6eade2b7fd24c10aae109da5e2e6718
15:11:05.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.052978 27967 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Leader pre-election lost for term 4. Reason: could not achieve majority
15:11:05.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.101506 27892 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.190:36655
15:11:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.107013 27892 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:11:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.106242 27983 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.190:36655 every 8 connection(s)
15:11:05.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.112133 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 27892
15:11:05.113 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:11:05.113 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.189:37765
15:11:05.113 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.118536 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/logs
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.189:37765
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.189
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45529
15:11:05.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.189
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:11:05.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:11:05.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.134624 27984 sys_catalog.cc:263] Verifying existing consensus state
15:11:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.143620 27984 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap starting.
15:11:05.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.202721 27984 log.cc:826] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Log is configured to *not* fsync() on all Append() calls
15:11:05.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.232651 27984 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:11:05.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.233763 27984 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap complete.
15:11:05.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.269362 27984 raft_consensus.cc:359] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:05.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.272737 27984 raft_consensus.cc:740] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86daac6df1334b1bb65f5108dcdd44cc, State: Initialized, Role: FOLLOWER
15:11:05.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.273924 27984 consensus_queue.cc:260] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:05.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.303579 27988 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.307788 27988 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:11:05.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.305900 27984 sys_catalog.cc:565] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: configured and running, proceeding with master startup.
15:11:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.422806 27999 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:11:05.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:05 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638664, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:11:05.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.484398 27892 master_runner.cc:186] Error getting master registration for 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111),
15:11:05.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.534291 27892 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:11:05.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.688613 27986 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:11:05.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.689229 27986 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:11:05.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.689924 27986 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:11:05.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.753695 27986 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:11:05.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.754022 27986 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:11:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.754551 27986 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:11:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.754927 27986 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:11:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.755254 27986 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:11:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.755607 27986 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:11:05.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.755851 27986 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.809718 27986 master_runner.cc:386] Master server non-default flags:
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.189
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.189:37765
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:11:05.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.189
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45529
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/logs
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:11:05.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:11:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:11:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.811399 27986 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:11:05.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:05.812631 27986 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:11:05.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.825285 28008 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.451103 27999 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:11:06.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.451489 27999 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: loading cluster ID for follower catalog manager: success
15:11:06.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.507121 27999 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: acquiring CA information for follower catalog manager: success
15:11:06.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.521729 27999 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:11:06.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:06 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638661, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.188@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:11:06.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.571965 27818 master_runner.cc:186] Error getting master registration for 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111),
15:11:06.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.572595 27818 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:11:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:06.786922 26233 debug-util.cc:398] Leaking SignalData structure 0x7b08000af3c0 after lost signal to thread 26170
15:11:06.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:06.788614 26233 debug-util.cc:398] Leaking SignalData structure 0x7b08000adfe0 after lost signal to thread 26236
15:11:06.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.843400 28018 raft_consensus.cc:493] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 3 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:11:06.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.844170 28018 raft_consensus.cc:515] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:06.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.863545 28018 leader_election.cc:290] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765), 65b363a0f05e4d7fa201cbc9853bb322 (127.26.64.188:32887)
15:11:06.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:06.890527 27931 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:11:06.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:06.909173 27931 leader_election.cc:336] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:11:06.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.949132 27865 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86daac6df1334b1bb65f5108dcdd44cc" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "65b363a0f05e4d7fa201cbc9853bb322" is_pre_election: true
15:11:06.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.950335 27865 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 86daac6df1334b1bb65f5108dcdd44cc in term 3.
15:11:06.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.954236 27931 leader_election.cc:304] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322, 86daac6df1334b1bb65f5108dcdd44cc; no voters: b6eade2b7fd24c10aae109da5e2e6718
15:11:06.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.955181 28018 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 3 FOLLOWER]: Leader pre-election won for term 4
15:11:06.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.955673 28018 raft_consensus.cc:493] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 3 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:11:06.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.956125 28018 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 3 FOLLOWER]: Advancing to term 4
15:11:06.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.974063 28018 raft_consensus.cc:515] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:06.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.976688 28018 leader_election.cc:290] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [CANDIDATE]: Term 4 election: Requested vote from peers b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765), 65b363a0f05e4d7fa201cbc9853bb322 (127.26.64.188:32887)
15:11:06.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.978178 27865 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "86daac6df1334b1bb65f5108dcdd44cc" candidate_term: 4 candidate_status { last_received { term: 3 index: 6 } } ignore_live_leader: false dest_uuid: "65b363a0f05e4d7fa201cbc9853bb322"
15:11:06.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:06.978775 27865 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 3 FOLLOWER]: Advancing to term 4
15:11:06.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:06.981853 27931 leader_election.cc:336] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:11:07.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.004377 27865 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 86daac6df1334b1bb65f5108dcdd44cc in term 4.
15:11:07.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.006839 27931 leader_election.cc:304] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322, 86daac6df1334b1bb65f5108dcdd44cc; no voters: b6eade2b7fd24c10aae109da5e2e6718
15:11:07.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.008041 28018 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 4 FOLLOWER]: Leader election won for term 4
15:11:07.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.010505 28018 raft_consensus.cc:697] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 4 LEADER]: Becoming Leader. State: Replica: 86daac6df1334b1bb65f5108dcdd44cc, State: Running, Role: LEADER
15:11:07.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.011647 28018 consensus_queue.cc:237] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:07.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.032053 28023 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 86daac6df1334b1bb65f5108dcdd44cc. Latest consensus state: current_term: 4 leader_uuid: "86daac6df1334b1bb65f5108dcdd44cc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:07.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.032869 28023 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: LEADER
15:11:07.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.047623 28025 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:11:07.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.056700 28025 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:11:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.068966 28025 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:11:07.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.069258 28025 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:11:07.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.078050 28025 catalog_manager.cc:1514] Loading token signing keys...
15:11:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.086337 28025 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Loaded TSK: 0
15:11:07.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.087656 28025 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:11:07.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.409940 27865 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Refusing update from remote peer 86daac6df1334b1bb65f5108dcdd44cc: Log matching property violated. Preceding OpId in replica: term: 3 index: 6. Preceding OpId from leader: term: 4 index: 7. (index mismatch)
15:11:07.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.411590 28023 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [LEADER]: Connected to new peer: Peer: permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
15:11:07.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.447110 28026 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 86daac6df1334b1bb65f5108dcdd44cc. Latest consensus state: current_term: 4 leader_uuid: "86daac6df1334b1bb65f5108dcdd44cc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:07.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.451459 28026 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: FOLLOWER
15:11:07.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.460597 28023 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "86daac6df1334b1bb65f5108dcdd44cc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:07.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.461531 28023 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: LEADER
15:11:07.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:07.462195 27931 consensus_peers.cc:597] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc -> Peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Couldn't send request to peer b6eade2b7fd24c10aae109da5e2e6718. Status: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:11:07.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.476302 28026 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "86daac6df1334b1bb65f5108dcdd44cc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:07.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.477229 28026 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: FOLLOWER
15:11:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:07.500622 26308 debug-util.cc:398] Leaking SignalData structure 0x7b0800099000 after lost signal to thread 26243
15:11:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:07.501391 26308 debug-util.cc:398] Leaking SignalData structure 0x7b08000970e0 after lost signal to thread 26311
15:11:07.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:05.828022 28009 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:07.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:07.228207 28007 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 27986
15:11:07.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:07.730347 27986 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.906s user 0.644s sys 1.111s
15:11:07.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:07.741886 27986 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.917s user 0.644s sys 1.111s
15:11:07.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.748133 27986 server_base.cc:1047] running on GCE node
15:11:07.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:07.748979 28032 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:07.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.750524 27986 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:11:07.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.760741 27986 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:11:07.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.762830 27986 hybrid_clock.cc:648] HybridClock initialized: now 1765638667762792 us; error 50 us; skew 500 ppm
15:11:07.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.764101 27986 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:11:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:07 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638667, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:11:07.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.773092 27986 init.cc:377] Logged in from keytab as kudu/127.26.64.189@KRBTEST.COM (short username kudu)
15:11:07.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.822757 27986 webserver.cc:492] Webserver started at http://127.26.64.189:45529/ using document root <none> and password file <none>
15:11:07.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.824102 27986 fs_manager.cc:362] Metadata directory not provided
15:11:07.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.824420 27986 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:11:07.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.845060 27986 fs_manager.cc:714] Time spent opening directory manager: real 0.016s user 0.008s sys 0.008s
15:11:07.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.860188 28038 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:11:07.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.861817 27986 fs_manager.cc:730] Time spent opening block manager: real 0.014s user 0.013s sys 0.000s
15:11:07.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.862263 27986 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:11:07.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6eade2b7fd24c10aae109da5e2e6718"
15:11:07.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:37 on dist-test-slave-xht5"
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.865372 27986 fs_report.cc:389] FS layout report
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/data
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:11:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:11:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.977133 27986 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:11:07.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.987460 27986 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:11:07.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:07.988091 27986 kserver.cc:163] Server-wide thread pool size limit: 3276
15:11:08.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:08.481950 27986 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.189:37765
15:11:08.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:08.482007 28091 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.189:37765 every 8 connection(s)
15:11:08.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:08.487242 27986 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:11:08.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:08.497720 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 27986
15:11:08.498 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:11:08.499 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.64.188:32887(127.26.64.188:32887)] connecting to peer
15:11:08.502 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.26.64.188:32887(127.26.64.188:32887)] Successfully connected to peer
15:11:08.538 [DEBUG - kudu-nio-0] (SecurityContext.java:222) Refreshing Kerberos credentials...
15:11:08.539 [DEBUG - kudu-nio-0] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc
15:11:08.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:08 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638658, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:11:08.549 [DEBUG - kudu-nio-0] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
15:11:08.550 [INFO - kudu-nio-0] (SecurityContext.java:256) Successfully refreshed Kerberos credentials from ticket cache
15:11:08.551 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.26.64.188
15:11:08.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:08 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638664, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for kudu/127.26.64.189@KRBTEST.COM
15:11:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:08 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638658, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.26.64.188@KRBTEST.COM
15:11:08.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:08.639084 28092 sys_catalog.cc:263] Verifying existing consensus state
15:11:08.688 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x25c8c735, L:/127.0.0.1:52692 - R:/127.26.64.188:32887] using SASL/GSSAPI
15:11:08.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:08.715739 28092 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Bootstrap starting.
15:11:08.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:08.917973 28092 log.cc:826] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Log is configured to *not* fsync() on all Append() calls
15:11:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:08.971446 28092 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Bootstrap replayed 1/1 log segments. Stats: ops{read=6 overwritten=0 applied=6 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:11:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:08.973136 28092 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Bootstrap complete.
15:11:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.030077 28092 raft_consensus.cc:359] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 3 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:09.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.037696 28092 raft_consensus.cc:740] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 3 FOLLOWER]: Becoming Follower/Learner. State: Replica: b6eade2b7fd24c10aae109da5e2e6718, State: Initialized, Role: FOLLOWER
15:11:09.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.039963 28092 consensus_queue.cc:260] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 6, Last appended: 3.6, Last appended by leader: 6, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:09.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.057298 28067 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [term 3 FOLLOWER]: Advancing to term 4
15:11:09.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.065905 28092 sys_catalog.cc:565] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: configured and running, proceeding with master startup.
15:11:09.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.053305 28099 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 3 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:09.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.072479 28099 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:11:09.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.103050 28099 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 86daac6df1334b1bb65f5108dcdd44cc. Latest consensus state: current_term: 4 leader_uuid: "86daac6df1334b1bb65f5108dcdd44cc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:09.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.103780 28099 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:11:09.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.105163 28101 sys_catalog.cc:455] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "86daac6df1334b1bb65f5108dcdd44cc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:09.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.105914 28101 sys_catalog.cc:458] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718 [sys.catalog]: This master's current role is: FOLLOWER
15:11:09.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.198711 28113 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
15:11:09.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:09 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638667, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:11:09.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:09 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638667, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for kudu/127.26.64.189@KRBTEST.COM
15:11:09.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:09 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638667, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for kudu/127.26.64.188@KRBTEST.COM
15:11:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.549278 28018 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "86daac6df1334b1bb65f5108dcdd44cc" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:09.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.550572 28018 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: LEADER
15:11:09.705 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.188:32887
15:11:09.705 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
15:11:09.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.706085 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 27818
15:11:09.743 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.26.64.188:32887(127.26.64.188:32887)] handling channelInactive
15:11:09.743 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.26.64.188:32887(127.26.64.188:32887)] cleaning up while in state READY due to: connection closed
15:11:09.748 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:11:09.749 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.190:36655
15:11:09.749 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
15:11:09.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.749465 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 27892
15:11:09.786 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:11:09.786 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.26.64.189:37765
15:11:09.787 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
15:11:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.787307 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 27986
15:11:09.818 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:11:09.818 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.188:32887
15:11:09.819 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:09.820811 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/logs
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:09.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.188:32887
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.188
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37067
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.188
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:11:09.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:11:10.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.145090 28121 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:11:10.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.145710 28121 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:11:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.146584 28121 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:11:10.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.181560 28121 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:11:10.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.181881 28121 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:11:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.182284 28121 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:11:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.182593 28121 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:11:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.182829 28121 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:11:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.183012 28121 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:11:10.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.183144 28121 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:11:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:10.224303 28121 master_runner.cc:386] Master server non-default flags:
15:11:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:10.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.188
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.188:32887
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:11:10.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.188
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37067
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/logs
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:10.225600 28121 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:11:10.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:10.226645 28121 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:11:10.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.237517 28128 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:11.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:10.239683 28129 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:11.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:11.558346 28131 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:11.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:11.560770 28130 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1317 milliseconds
15:11:11.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.560942 28121 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:11:11.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.562268 28121 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:11:11.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.564813 28121 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:11:11.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.566151 28121 hybrid_clock.cc:648] HybridClock initialized: now 1765638671566106 us; error 57 us; skew 500 ppm
15:11:11.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.567014 28121 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:11:11.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:11 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638671, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:11:11.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.573179 28121 init.cc:377] Logged in from keytab as kudu/127.26.64.188@KRBTEST.COM (short username kudu)
15:11:11.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.579222 28121 webserver.cc:492] Webserver started at http://127.26.64.188:37067/ using document root <none> and password file <none>
15:11:11.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.580261 28121 fs_manager.cc:362] Metadata directory not provided
15:11:11.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.580487 28121 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:11:11.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.588704 28121 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.001s sys 0.004s
15:11:11.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.594043 28140 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:11:11.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.595356 28121 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
15:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.595660 28121 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65b363a0f05e4d7fa201cbc9853bb322"
15:11:11.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:39 on dist-test-slave-xht5"
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.597606 28121 fs_report.cc:389] FS layout report
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/wal
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/data
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:11:11.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:11:11.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.651958 28121 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:11:11.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.664510 28121 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:11:11.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.664965 28121 kserver.cc:163] Server-wide thread pool size limit: 3276
15:11:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.741464 28121 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.188:32887
15:11:11.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.741537 28191 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.188:32887 every 8 connection(s)
15:11:11.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.744275 28121 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-2/data/info.pb
15:11:11.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.746698 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 28121
15:11:11.748 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:11:11.748 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.190:36655
15:11:11.748 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.750036 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/logs
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:11:11.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.190:36655
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.190
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44727
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.190
15:11:11.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:11:11.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:11:11.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.756749 28192 sys_catalog.cc:263] Verifying existing consensus state
15:11:11.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.766726 28192 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap starting.
15:11:11.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.822395 28192 log.cc:826] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Log is configured to *not* fsync() on all Append() calls
15:11:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.846259 28192 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:11:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.847246 28192 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Bootstrap complete.
15:11:11.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.871178 28192 raft_consensus.cc:359] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:11.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.873600 28192 raft_consensus.cc:740] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 65b363a0f05e4d7fa201cbc9853bb322, State: Initialized, Role: FOLLOWER
15:11:11.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.874608 28192 consensus_queue.cc:260] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:11.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.877170 28196 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:11.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.878168 28196 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: FOLLOWER
15:11:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.878767 28192 sys_catalog.cc:565] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: configured and running, proceeding with master startup.
15:11:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.910737 28207 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:11:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.911243 28207 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: loading cluster ID for follower catalog manager: success
15:11:11.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.919215 28207 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: acquiring CA information for follower catalog manager: success
15:11:11.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.925021 28207 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:11:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.907341 28121 master_runner.cc:186] Error getting master registration for 127.26.64.190:36655: Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111),
15:11:11.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:11.939298 28121 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:11:12.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.115962 28194 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:11:12.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.116564 28194 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:11:12.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.117298 28194 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:11:12.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.152107 28194 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:11:12.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.152424 28194 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:11:12.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.152864 28194 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:11:12.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.153143 28194 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:11:12.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.153421 28194 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:11:12.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.153697 28194 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:11:12.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.153913 28194 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:12.192966 28194 master_runner.cc:386] Master server non-default flags:
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.190
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.190:36655
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:12.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.190
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44727
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/logs
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:11:12.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:12.194707 28194 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:11:12.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:12.195760 28194 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:11:12.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.208910 28215 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:13.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:13.275610 28223 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:11:13.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:13.276861 28223 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:13.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:13.284942 28223 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:11:13.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:13.309864 28142 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111)
15:11:13.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:13.324409 28142 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655): Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111)
15:11:13.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:13.326593 28142 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:11:13.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:13.326982 28142 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322; no voters: 86daac6df1334b1bb65f5108dcdd44cc, b6eade2b7fd24c10aae109da5e2e6718
15:11:13.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:13.327831 28223 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
15:11:14.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:12.213680 28216 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:14.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:13.612143 28214 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 28194
15:11:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.176025 28194 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.963s user 0.571s sys 0.898s
15:11:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.178289 28218 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:14.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.178623 28194 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.965s user 0.572s sys 0.898s
15:11:14.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.185132 28217 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1971 milliseconds
15:11:14.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.185225 28194 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:11:14.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.186800 28194 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:11:14.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.189908 28194 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:11:14.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.191362 28194 hybrid_clock.cc:648] HybridClock initialized: now 1765638674191331 us; error 34 us; skew 500 ppm
15:11:14.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.192469 28194 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:11:14.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:14 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638674, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:11:14.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.200140 28194 init.cc:377] Logged in from keytab as kudu/127.26.64.190@KRBTEST.COM (short username kudu)
15:11:14.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.206041 28194 webserver.cc:492] Webserver started at http://127.26.64.190:44727/ using document root <none> and password file <none>
15:11:14.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.207259 28194 fs_manager.cc:362] Metadata directory not provided
15:11:14.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.207520 28194 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:11:14.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.217989 28194 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
15:11:14.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.226327 28231 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:11:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.227804 28194 fs_manager.cc:730] Time spent opening block manager: real 0.007s user 0.003s sys 0.004s
15:11:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.228230 28194 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:11:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "86daac6df1334b1bb65f5108dcdd44cc"
15:11:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:35 on dist-test-slave-xht5"
15:11:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.231132 28194 fs_report.cc:389] FS layout report
15:11:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:11:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:11:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/wal
15:11:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/data
15:11:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:11:14.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:11:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:11:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:11:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:11:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:11:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:11:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:11:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:11:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:11:14.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:11:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.289808 28194 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:11:14.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.295675 28194 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:11:14.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.296084 28194 kserver.cc:163] Server-wide thread pool size limit: 3276
15:11:14.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.388092 28194 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.190:36655
15:11:14.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.388139 28282 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.190:36655 every 8 connection(s)
15:11:14.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.391096 28194 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-0/data/info.pb
15:11:14.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.397616 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 28194
15:11:14.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:11:14.398 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.26.64.189:37765
15:11:14.398 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
15:11:14.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.400329 26882 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:11:14.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/logs
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.189:37765
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.189
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45529
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.189
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:14.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:11:14.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
15:11:14.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
15:11:14.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.403772 28283 sys_catalog.cc:263] Verifying existing consensus state
15:11:14.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.411635 28283 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap starting.
15:11:14.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.452886 28283 log.cc:826] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Log is configured to *not* fsync() on all Append() calls
15:11:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.472808 28283 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
15:11:14.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.473724 28283 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: Bootstrap complete.
15:11:14.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.498603 28283 raft_consensus.cc:359] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 4 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:14.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.501458 28283 raft_consensus.cc:740] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 4 FOLLOWER]: Becoming Follower/Learner. State: Replica: 86daac6df1334b1bb65f5108dcdd44cc, State: Initialized, Role: FOLLOWER
15:11:14.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.502537 28283 consensus_queue.cc:260] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:14.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.511523 28287 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 4 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.513178 28287 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:11:14.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.513628 28283 sys_catalog.cc:565] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: configured and running, proceeding with master startup.
15:11:14.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.562294 28298 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:11:14.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.562932 28298 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: loading cluster ID for follower catalog manager: success
15:11:14.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.579077 28298 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: acquiring CA information for follower catalog manager: success
15:11:14.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.584852 28298 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
15:11:14.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:14 dist-test-slave-xht5 krb5kdc[26893](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638674, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.190@KRBTEST.COM for kudu/127.26.64.190@KRBTEST.COM
15:11:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.639132 28194 master_runner.cc:186] Error getting master registration for 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111),
15:11:14.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:14.673704 28194 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:11:14.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.921319 28285 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
15:11:14.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.922075 28285 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
15:11:14.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.923189 28285 flags.cc:425] Enabled unsafe flag: --never_fsync=true
15:11:14.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.982786 28285 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
15:11:14.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.983155 28285 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
15:11:14.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.983654 28285 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
15:11:14.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.983999 28285 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
15:11:14.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.984390 28285 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
15:11:14.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.984735 28285 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
15:11:14.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:14.984984 28285 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:36497/jwks.json
15:11:15.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.054822 28285 master_runner.cc:386] Master server non-default flags:
15:11:15.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
15:11:15.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.64.148:35829
15:11:15.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
15:11:15.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
15:11:15.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data
15:11:15.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.64.190:36655,127.26.64.189:37765,127.26.64.188:32887
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/kudu.keytab
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.26.64.189
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.64.189:37765
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:36497/jwks.json
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.64.189
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=45529
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/logs
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
15:11:15.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
15:11:15.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
15:11:15.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
15:11:15.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
15:11:15.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 13 Dec 2025 14:43:22 UTC on 5fd53c4cbb9d
15:11:15.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9497
15:11:15.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
15:11:15.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.056818 28285 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
15:11:15.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.058465 28285 file_cache.cc:492] Constructed file cache file cache with capacity 419430
15:11:15.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:15.073999 28307 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:15.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.524804 28314 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
15:11:15.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.525300 28314 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:15.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.528972 28314 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:11:15.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:15.575121 28142 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:11:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.598358 28258 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "65b363a0f05e4d7fa201cbc9853bb322" candidate_term: 5 candidate_status { last_received { term: 4 index: 7 } } ignore_live_leader: false dest_uuid: "86daac6df1334b1bb65f5108dcdd44cc" is_pre_election: true
15:11:15.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.599603 28258 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 4 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 65b363a0f05e4d7fa201cbc9853bb322 in term 4.
15:11:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.602610 28142 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322, 86daac6df1334b1bb65f5108dcdd44cc; no voters: b6eade2b7fd24c10aae109da5e2e6718
15:11:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.604184 28314 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Leader pre-election won for term 5
15:11:15.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.604641 28314 raft_consensus.cc:493] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
15:11:15.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.605026 28314 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 4 FOLLOWER]: Advancing to term 5
15:11:15.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.636070 28314 raft_consensus.cc:515] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 5 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:15.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.643023 28258 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "65b363a0f05e4d7fa201cbc9853bb322" candidate_term: 5 candidate_status { last_received { term: 4 index: 7 } } ignore_live_leader: false dest_uuid: "86daac6df1334b1bb65f5108dcdd44cc"
15:11:15.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.643926 28258 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 4 FOLLOWER]: Advancing to term 5
15:11:15.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:15.664254 28142 leader_election.cc:336] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 5 election: RPC error from VoteRequest() call to peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111)
15:11:15.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.660410 28314 leader_election.cc:290] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 5 election: Requested vote from peers 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655), b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765)
15:11:15.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.674274 28258 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 5 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 65b363a0f05e4d7fa201cbc9853bb322 in term 5.
15:11:15.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.678727 28142 leader_election.cc:304] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [CANDIDATE]: Term 5 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 65b363a0f05e4d7fa201cbc9853bb322, 86daac6df1334b1bb65f5108dcdd44cc; no voters: b6eade2b7fd24c10aae109da5e2e6718
15:11:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.680953 28314 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 5 FOLLOWER]: Leader election won for term 5
15:11:15.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.696650 28314 raft_consensus.cc:697] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [term 5 LEADER]: Becoming Leader. State: Replica: 65b363a0f05e4d7fa201cbc9853bb322, State: Running, Role: LEADER
15:11:15.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.698371 28314 consensus_queue.cc:237] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 4.7, Last appended by leader: 7, Current term: 5, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } }
15:11:15.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.747681 28319 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 65b363a0f05e4d7fa201cbc9853bb322. Latest consensus state: current_term: 5 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:15.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.753708 28319 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: LEADER
15:11:15.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.778007 28321 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
15:11:15.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.828354 28321 catalog_manager.cc:1494] Initializing Kudu cluster ID...
15:11:15.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:15.931296 26345 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.25.139.125:44983, user_credentials={real_user=slave}} blocked reactor thread for 47749.9us
15:11:15.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.938506 28321 catalog_manager.cc:1269] Loaded cluster ID: 369b7cafdbc546759fa24083ca431298
15:11:15.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.946867 28321 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
15:11:16.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:15.995263 28321 catalog_manager.cc:1514] Loading token signing keys...
15:11:16.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.025451 28321 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322: Loaded TSK: 0
15:11:16.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.027907 28321 catalog_manager.cc:1524] Initializing in-progress tserver states...
15:11:16.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.220800 28258 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [term 5 FOLLOWER]: Refusing update from remote peer 65b363a0f05e4d7fa201cbc9853bb322: Log matching property violated. Preceding OpId in replica: term: 4 index: 7. Preceding OpId from leader: term: 5 index: 8. (index mismatch)
15:11:16.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.223467 28319 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [LEADER]: Connected to new peer: Peer: permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.001s
15:11:16.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.269958 28324 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: New leader 65b363a0f05e4d7fa201cbc9853bb322. Latest consensus state: current_term: 5 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:16.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.270798 28324 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:11:16.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.284778 28324 sys_catalog.cc:455] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 5 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:16.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.285503 28324 sys_catalog.cc:458] T 00000000000000000000000000000000 P 86daac6df1334b1bb65f5108dcdd44cc [sys.catalog]: This master's current role is: FOLLOWER
15:11:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.289644 28319 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 5 leader_uuid: "65b363a0f05e4d7fa201cbc9853bb322" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "86daac6df1334b1bb65f5108dcdd44cc" member_type: VOTER last_known_addr { host: "127.26.64.190" port: 36655 } } peers { permanent_uuid: "b6eade2b7fd24c10aae109da5e2e6718" member_type: VOTER last_known_addr { host: "127.26.64.189" port: 37765 } } peers { permanent_uuid: "65b363a0f05e4d7fa201cbc9853bb322" member_type: VOTER last_known_addr { host: "127.26.64.188" port: 32887 } } }
15:11:16.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.290549 28319 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 [sys.catalog]: This master's current role is: LEADER
15:11:16.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:16.337198 28142 consensus_peers.cc:597] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 -> Peer b6eade2b7fd24c10aae109da5e2e6718 (127.26.64.189:37765): Couldn't send request to peer b6eade2b7fd24c10aae109da5e2e6718. Status: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:11:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.955252 28121 master_runner.cc:186] Error getting master registration for 127.26.64.189:37765: Network error: Client connection negotiation failed: client connection to 127.26.64.189:37765: connect: Connection refused (error 111),
15:11:16.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:16.955902 28121 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
15:11:18.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:15.077597 28308 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:18.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:16.475209 28306 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 28285
15:11:18.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:18.380019 28285 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.303s user 1.173s sys 1.939s
15:11:18.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:18.381340 28285 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.305s user 1.173s sys 1.940s
15:11:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:18.381995 28309 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3303 milliseconds
15:11:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.382498 28285 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
15:11:18.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:18.382561 28310 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
15:11:18.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.387517 28285 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
15:11:18.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.392452 28285 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
15:11:18.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.393935 28285 hybrid_clock.cc:648] HybridClock initialized: now 1765638678393778 us; error 158 us; skew 500 ppm
15:11:18.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.395088 28285 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
15:11:18.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 13 15:11:18 dist-test-slave-xht5 krb5kdc[26893](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765638678, etypes {rep=17 tkt=17 ses=17}, kudu/127.26.64.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
15:11:18.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.401162 28285 init.cc:377] Logged in from keytab as kudu/127.26.64.189@KRBTEST.COM (short username kudu)
15:11:18.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.409284 28285 webserver.cc:492] Webserver started at http://127.26.64.189:45529/ using document root <none> and password file <none>
15:11:18.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.410576 28285 fs_manager.cc:362] Metadata directory not provided
15:11:18.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.410849 28285 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
15:11:18.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.421144 28285 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.001s sys 0.005s
15:11:18.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.425658 28346 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
15:11:18.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.426920 28285 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.427286 28285 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data,/tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b6eade2b7fd24c10aae109da5e2e6718"
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-13 15:10:37 on dist-test-slave-xht5"
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.429172 28285 fs_report.cc:389] FS layout report
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/wal
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/data
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
15:11:18.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
15:11:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
15:11:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
15:11:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
15:11:18.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
15:11:18.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.479110 28285 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
15:11:18.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.488271 28285 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
15:11:18.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.488740 28285 kserver.cc:163] Server-wide thread pool size limit: 3276
15:11:18.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.562659 28285 rpc_server.cc:307] RPC server started. Bound to: 127.26.64.189:37765
15:11:18.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.562734 28397 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.64.189:37765 every 8 connection(s)
15:11:18.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.565480 28285 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/master-1/data/info.pb
15:11:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.571481 26882 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu as pid 28285
15:11:18.572 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
15:11:18.572 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.26.64.188:32887(127.26.64.188:32887)] connecting to peer
15:11:18.575 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.26.64.188:32887(127.26.64.188:32887)] Successfully connected to peer
15:11:18.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.579722 28398 sys_catalog.cc:263] Verifying existing consensus state
15:11:18.589 [DEBUG - kudu-nio-1] (SecurityContext.java:222) Refreshing Kerberos credentials...
15:11:18.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.587734 28398 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Bootstrap starting.
15:11:18.590 [DEBUG - kudu-nio-1] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskrL_IJG/test-tmp/mini-kudu-cluster494242758286999945/krb5kdc/krb5cc
15:11:18.593 [DEBUG - kudu-nio-1] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
15:11:18.593 [WARN - kudu-nio-1] (SecurityContext.java:231) Tried to refresh Kerberos credentials but was unable to re-login from ticket cache
15:11:18.593 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.26.64.188:32887(127.26.64.188:32887)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
15:11:18.595 [DEBUG - main] (Connection.java:575) [peer master-127.26.64.190:36655(127.26.64.190:36655)] disconnecting while in state TERMINATED
15:11:18.596 [DEBUG - main] (Connection.java:575) [peer master-127.26.64.189:37765(127.26.64.189:37765)] disconnecting while in state TERMINATED
15:11:18.596 [DEBUG - main] (Connection.java:575) [peer master-127.26.64.188:32887(127.26.64.188:32887)] disconnecting while in state TERMINATED
15:11:18.596 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.26.64.188:32887(127.26.64.188:32887)] handling channelInactive
15:11:18.597 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:11:18.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:18.602982 28400 negotiation.cc:337] Failed RPC negotiation. Trace:
15:11:18.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:11:18.581710 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:53182 (local address 127.26.64.188:32887)
15:11:18.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:11:18.586557 (+ 4847us) server_negotiation.cc:207] Beginning negotiation
15:11:18.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:11:18.586591 (+ 34us) server_negotiation.cc:400] Waiting for connection header
15:11:18.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:11:18.586653 (+ 62us) server_negotiation.cc:408] Connection header received
15:11:18.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:11:18.586817 (+ 164us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
15:11:18.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:11:18.586851 (+ 34us) server_negotiation.cc:462] Received NEGOTIATE request from client
15:11:18.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:11:18.587250 (+ 399us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
15:11:18.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:11:18.588145 (+ 895us) server_negotiation.cc:227] Negotiated authn=SASL
15:11:18.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1213 15:11:18.600680 (+ 12535us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:53182: BlockingRecv error: recv got EOF from 127.0.0.1:53182 (error 108)
15:11:18.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":4171,"thread_start_us":3856,"threads_started":1}
15:11:18.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.603686 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 28194
15:11:18.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.652493 28398 log.cc:826] T 00000000000000000000000000000000 P b6eade2b7fd24c10aae109da5e2e6718: Log is configured to *not* fsync() on all Append() calls
15:11:18.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:18.657024 28142 connection.cc:537] client connection to 127.26.64.190:36655 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
15:11:18.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:18.657512 28142 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107) [suppressed 8 similar messages]
15:11:18.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.657693 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 28285
15:11:18.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:18.661706 28142 consensus_peers.cc:597] T 00000000000000000000000000000000 P 65b363a0f05e4d7fa201cbc9853bb322 -> Peer 86daac6df1334b1bb65f5108dcdd44cc (127.26.64.190:36655): Couldn't send request to peer 86daac6df1334b1bb65f5108dcdd44cc. Status: Network error: Client connection negotiation failed: client connection to 127.26.64.190:36655: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
15:11:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:18.675948 26882 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 28121
15:11:19.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:19.605373 26390 debug-util.cc:398] Leaking SignalData structure 0x7b08000aaea0 after lost signal to thread 26322
15:11:19.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251213 15:11:19.605955 26390 debug-util.cc:398] Leaking SignalData structure 0x7b080006f7a0 after lost signal to thread 26393
15:11:19.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:11:19Z chronyd exiting
15:11:20.944 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
15:11:20.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:20.946012 26157 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 26456
15:11:20.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:20.986663 26157 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 26592
15:11:21.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:21.012837 26157 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 26741
15:11:21.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:21.041464 26157 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 26169
15:11:21.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:21.075824 26157 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 26239
15:11:21.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251213 15:11:21.110087 26157 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskrL_IJG/build/tsan/bin/kudu with pid 26314
15:11:22.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-13T15:11:22Z chronyd exiting
15:11:23.276 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565) ~[main/:?]
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367) ~[main/:?]
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94) ~[main/:?]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testRenewAndReacquireKeberosCredentials(TestSecurity.java:346)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more