21:03:11.792 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:03:11.806 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskerlxO1/test-tmp
21:03:11.812 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:03:11.815 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskerlxO1/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:03:12.053 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:03:12.233 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:03:12.235 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:03:12.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:12.240558 27552 mini_oidc.cc:150] Starting JWKS server
21:03:12.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:12.244868 27552 webserver.cc:577] Webserver started at http://127.0.0.1:46437/ using document root <none> and password file <none>
21:03:12.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:12.245508 27552 mini_oidc.cc:186] Starting OIDC Discovery server
21:03:12.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:12.247689 27552 webserver.cc:577] Webserver started at http://127.0.0.1:42817/ using document root <none> and password file <none>
21:03:12.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-14T21:03:12Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:03:12.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-14T21:03:12Z Disabled control of system clock
21:03:12.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:12.310818 27552 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:12.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:12.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/wal
21:03:12.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/data
21:03:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:12.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/logs
21:03:12.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/data/info.pb
21:03:12.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:12.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:12.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.62:39275
21:03:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.62
21:03:12.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:12.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:12.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:12.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:12.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46437/jwks.json with env {}
21:03:12.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:12.624126 27565 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:12.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:12.624708 27565 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:12.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:12.625284 27565 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:12.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:12.659052 27565 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:03:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:12.659385 27565 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:12.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:12.659734 27565 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:12.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:12.659953 27565 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:03:12.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:12.660224 27565 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:12.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:12.660419 27565 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46437/jwks.json
21:03:12.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:12.700259 27565 master_runner.cc:387] Master server non-default flags:
21:03:12.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/logs
21:03:12.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:12.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:12.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:12.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:12.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:12.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/data
21:03:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/wal
21:03:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:12.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.62:39275
21:03:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:12.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46437/jwks.json
21:03:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/data/info.pb
21:03:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.62
21:03:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:12.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:12.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:12.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:12.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:12.701792 27565 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:12.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:12.704649 27565 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:12.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:12.718822 27571 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:14.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.123364 27570 debug-util.cc:398] Leaking SignalData structure 0x7b0800029240 after lost signal to thread 27565
21:03:14.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.168903 27565 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.450s user 0.483s sys 0.958s
21:03:14.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:12.719434 27572 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:14.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.169349 27565 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.451s user 0.483s sys 0.958s
21:03:14.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.171816 27574 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:14.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.173882 27573 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1450 milliseconds
21:03:14.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.173867 27565 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:03:14.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.175261 27565 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:14.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.178319 27565 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:14.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.180218 27565 hybrid_clock.cc:648] HybridClock initialized: now 1794690194180169 us; error 40 us; skew 500 ppm
21:03:14.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.181435 27565 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:14.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.188007 27565 webserver.cc:577] Webserver started at http://127.26.232.62:41899/ using document root <none> and password file <none>
21:03:14.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.188899 27565 fs_manager.cc:362] Metadata directory not provided
21:03:14.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.189085 27565 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:14.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.189637 27565 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:14.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.193979 27565 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/data/instance:
21:03:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e875411d7094b09bf311957bd5d4218"
21:03:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:14 on dist-test-slave-q92b"
21:03:14.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.195170 27565 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/wal/instance:
21:03:14.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e875411d7094b09bf311957bd5d4218"
21:03:14.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:14 on dist-test-slave-q92b"
21:03:14.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.202479 27565 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
21:03:14.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.208223 27581 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:14.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.209585 27565 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.002s
21:03:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.209875 27565 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/wal
21:03:14.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6e875411d7094b09bf311957bd5d4218"
21:03:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:14 on dist-test-slave-q92b"
21:03:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.210300 27565 fs_report.cc:389] FS layout report
21:03:14.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/wal
21:03:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/wal
21:03:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/data/data
21:03:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:14.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:14.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.279296 27565 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:14.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.280987 27565 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:14.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.281414 27565 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:14.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.353438 27565 rpc_server.cc:307] RPC server started. Bound to: 127.26.232.62:39275
21:03:14.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.353530 27632 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.232.62:39275 every 8 connection(s)
21:03:14.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.356362 27565 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/data/info.pb
21:03:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.359167 27552 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 27565
21:03:14.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.359721 27552 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-0/wal/instance
21:03:14.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.364478 27633 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:14.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.365020 27552 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:14.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:14.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/wal
21:03:14.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/data
21:03:14.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:14.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:14.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/logs
21:03:14.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/data/info.pb
21:03:14.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:14.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:14.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:14.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:14.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:14.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:14.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:14.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:14.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:14.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.61:37807
21:03:14.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.61
21:03:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:14.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:14.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46437/jwks.json with env {}
21:03:14.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.383623 27633 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } has no permanent_uuid. Determining permanent_uuid...
21:03:14.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.413309 27633 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } has no permanent_uuid. Determining permanent_uuid...
21:03:14.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.416128 27583 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.232.61:37807: connect: Connection refused (error 111)
21:03:14.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.420396 27633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.61:37807: Network error: Client connection negotiation failed: client connection to 127.26.232.61:37807: connect: Connection refused (error 111)
21:03:14.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.469894 27633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } attempt: 1
21:03:14.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.474637 27633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.61:37807: Network error: Client connection negotiation failed: client connection to 127.26.232.61:37807: connect: Connection refused (error 111)
21:03:14.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.543134 27633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } attempt: 2
21:03:14.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.547602 27633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.61:37807: Network error: Client connection negotiation failed: client connection to 127.26.232.61:37807: connect: Connection refused (error 111)
21:03:14.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.639101 27633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } attempt: 3
21:03:14.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.643810 27633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.61:37807: Network error: Client connection negotiation failed: client connection to 127.26.232.61:37807: connect: Connection refused (error 111)
21:03:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.703766 27635 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:14.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.704413 27635 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:14.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.704882 27635 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:14.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.738626 27635 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:03:14.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.738941 27635 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:14.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.739146 27635 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:14.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.739356 27635 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:03:14.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.739637 27635 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:14.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.739804 27635 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46437/jwks.json
21:03:14.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.780179 27635 master_runner.cc:387] Master server non-default flags:
21:03:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/logs
21:03:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:14.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/data
21:03:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/wal
21:03:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.61:37807
21:03:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46437/jwks.json
21:03:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:14.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/data/info.pb
21:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.61
21:03:14.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:14.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:14.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:14.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:14.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:14.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:14.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:14.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.781626 27635 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.783418 27635 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:14.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:14.787248 27633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } attempt: 4
21:03:14.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.792738 27633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.61:37807: Network error: Client connection negotiation failed: client connection to 127.26.232.61:37807: connect: Connection refused (error 111)
21:03:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.797971 27644 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:15.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:15.092243 27633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } attempt: 5
21:03:15.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:15.102684 27633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.61:37807: Network error: Client connection negotiation failed: client connection to 127.26.232.61:37807: connect: Connection refused (error 111)
21:03:15.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:15.650233 27633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } attempt: 6
21:03:15.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:15.681593 27633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.61:37807: Network error: Client connection negotiation failed: client connection to 127.26.232.61:37807: connect: Connection refused (error 111)
21:03:16.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:16.210860 27643 debug-util.cc:398] Leaking SignalData structure 0x7b0800029240 after lost signal to thread 27635
21:03:16.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:14.799881 27645 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:16.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:16.678063 27635 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.879s user 0.719s sys 1.159s
21:03:16.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:16.679641 27635 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.880s user 0.719s sys 1.159s
21:03:16.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:16.680748 27647 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:16.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:16.683380 27646 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1879 milliseconds
21:03:16.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.683447 27635 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:03:16.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.684653 27635 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:16.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.687047 27635 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:16.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.688555 27635 hybrid_clock.cc:648] HybridClock initialized: now 1794690196688519 us; error 32 us; skew 500 ppm
21:03:16.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.689625 27635 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:16.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.695611 27635 webserver.cc:577] Webserver started at http://127.26.232.61:35973/ using document root <none> and password file <none>
21:03:16.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.696619 27635 fs_manager.cc:362] Metadata directory not provided
21:03:16.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.696872 27635 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:16.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.697475 27635 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:16.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.701807 27635 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/data/instance:
21:03:16.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a770a96337fb4b68b5199c7d22074556"
21:03:16.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:16 on dist-test-slave-q92b"
21:03:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.703136 27635 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/wal/instance:
21:03:16.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a770a96337fb4b68b5199c7d22074556"
21:03:16.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:16 on dist-test-slave-q92b"
21:03:16.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.710695 27635 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
21:03:16.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.716465 27656 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.717726 27635 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
21:03:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.717993 27635 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/wal
21:03:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a770a96337fb4b68b5199c7d22074556"
21:03:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:16 on dist-test-slave-q92b"
21:03:16.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.718345 27635 fs_report.cc:389] FS layout report
21:03:16.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:16.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/wal
21:03:16.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/wal
21:03:16.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/data/data
21:03:16.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:16.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:16.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:16.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:16.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:16.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:16.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:16.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:16.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:16.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:16.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:16.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.742239 27633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } attempt: 7
21:03:16.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:16.749298 27633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.61:37807: Network error: Client connection negotiation failed: client connection to 127.26.232.61:37807: connect: Connection refused (error 111)
21:03:16.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.775825 27635 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:16.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.777491 27635 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:16.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.777947 27635 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:16.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.848474 27635 rpc_server.cc:307] RPC server started. Bound to: 127.26.232.61:37807
21:03:16.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.848556 27709 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.232.61:37807 every 8 connection(s)
21:03:16.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.851287 27635 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/data/info.pb
21:03:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.857717 27552 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 27635
21:03:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.858188 27552 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-1/wal/instance
21:03:16.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.858439 27710 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:16.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.861135 27552 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:16.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:16.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/wal
21:03:16.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/data
21:03:16.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:16.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:16.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:16.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:16.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:16.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:16.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/logs
21:03:16.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/data/info.pb
21:03:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:16.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:16.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:16.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:16.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.60:35871
21:03:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.60
21:03:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:16.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:16.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46437/jwks.json with env {}
21:03:16.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.872932 27710 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } has no permanent_uuid. Determining permanent_uuid...
21:03:16.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.895756 27710 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } has no permanent_uuid. Determining permanent_uuid...
21:03:16.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.910516 27710 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } has no permanent_uuid. Determining permanent_uuid...
21:03:16.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:16.912959 27661 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.26.232.60:35871: connect: Connection refused (error 111)
21:03:16.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:16.917486 27710 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.60:35871: Network error: Client connection negotiation failed: client connection to 127.26.232.60:35871: connect: Connection refused (error 111)
21:03:16.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:16.967015 27710 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } attempt: 1
21:03:16.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:16.971432 27710 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.60:35871: Network error: Client connection negotiation failed: client connection to 127.26.232.60:35871: connect: Connection refused (error 111)
21:03:17.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:17.039944 27710 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } attempt: 2
21:03:17.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.044240 27710 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.60:35871: Network error: Client connection negotiation failed: client connection to 127.26.232.60:35871: connect: Connection refused (error 111)
21:03:17.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:17.135722 27710 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } attempt: 3
21:03:17.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.140532 27710 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.60:35871: Network error: Client connection negotiation failed: client connection to 127.26.232.60:35871: connect: Connection refused (error 111)
21:03:17.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.184577 27712 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:17.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.185179 27712 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:17.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.185825 27712 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.218833 27712 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:03:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.219139 27712 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:17.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.219434 27712 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:17.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.219689 27712 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:03:17.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.219981 27712 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:17.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.220213 27712 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:46437/jwks.json
21:03:17.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:17.258713 27712 master_runner.cc:387] Master server non-default flags:
21:03:17.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/logs
21:03:17.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:17.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/data
21:03:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/wal
21:03:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:17.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:17.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.60:35871
21:03:17.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:17.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:17.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:17.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:46437/jwks.json
21:03:17.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:17.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:17.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/data/info.pb
21:03:17.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.60
21:03:17.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:17.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:17.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:17.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:17.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:17.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:17.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:17.260130 27712 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:17.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:17.262374 27712 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:17.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.275573 27722 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:17.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:17.283950 27710 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } attempt: 4
21:03:17.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.290786 27710 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.60:35871: Network error: Client connection negotiation failed: client connection to 127.26.232.60:35871: connect: Connection refused (error 111)
21:03:17.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:17.590188 27710 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } attempt: 5
21:03:17.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.605448 27710 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.60:35871: Network error: Client connection negotiation failed: client connection to 127.26.232.60:35871: connect: Connection refused (error 111)
21:03:18.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.153095 27710 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } attempt: 6
21:03:18.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:18.179494 27710 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.60:35871: Network error: Client connection negotiation failed: client connection to 127.26.232.60:35871: connect: Connection refused (error 111)
21:03:18.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.839751 27633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } attempt: 8
21:03:18.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:18.680969 27721 debug-util.cc:398] Leaking SignalData structure 0x7b0800029240 after lost signal to thread 27712
21:03:18.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:18.860600 27712 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.585s user 0.556s sys 1.021s
21:03:18.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:18.860967 27712 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.585s user 0.556s sys 1.021s
21:03:18.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:17.276132 27723 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:18.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:18.862172 27724 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1585 milliseconds
21:03:18.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:18.864115 27725 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:18.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.864850 27712 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:03:18.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.867306 27712 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:18.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.870747 27712 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:18.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.872457 27712 hybrid_clock.cc:648] HybridClock initialized: now 1794690198872391 us; error 49 us; skew 500 ppm
21:03:18.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.874301 27712 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.882754 27633 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } has no permanent_uuid. Determining permanent_uuid...
21:03:18.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.883024 27712 webserver.cc:577] Webserver started at http://127.26.232.60:35705/ using document root <none> and password file <none>
21:03:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.884298 27712 fs_manager.cc:362] Metadata directory not provided
21:03:18.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.884608 27712 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:18.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.885291 27712 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:18.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:18.889758 27633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.60:35871: Network error: Client connection negotiation failed: client connection to 127.26.232.60:35871: connect: Connection refused (error 111)
21:03:18.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.890708 27712 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/data/instance:
21:03:18.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661fad8838a84ba4a9f47ce4a78c19e8"
21:03:18.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:18 on dist-test-slave-q92b"
21:03:18.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.892081 27712 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/wal/instance:
21:03:18.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661fad8838a84ba4a9f47ce4a78c19e8"
21:03:18.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:18 on dist-test-slave-q92b"
21:03:18.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.899389 27712 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.004s sys 0.005s
21:03:18.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.905014 27737 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:18.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.906289 27712 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
21:03:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.906636 27712 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/wal
21:03:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "661fad8838a84ba4a9f47ce4a78c19e8"
21:03:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:18 on dist-test-slave-q92b"
21:03:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.907052 27712 fs_report.cc:389] FS layout report
21:03:18.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:18.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/wal
21:03:18.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/wal
21:03:18.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/data/data
21:03:18.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:18.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:18.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:18.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:18.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:18.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:18.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:18.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:18.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:18.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:18.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:18.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.955199 27633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } attempt: 1
21:03:18.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:18.959813 27633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.60:35871: Network error: Client connection negotiation failed: client connection to 127.26.232.60:35871: connect: Connection refused (error 111)
21:03:18.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.980784 27712 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.982846 27712 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:18.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:18.983410 27712 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:19.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.013286 27633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } attempt: 2
21:03:19.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:19.018231 27633 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.26.232.60:35871: Network error: Client connection negotiation failed: client connection to 127.26.232.60:35871: connect: Connection refused (error 111)
21:03:19.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.056174 27712 rpc_server.cc:307] RPC server started. Bound to: 127.26.232.60:35871
21:03:19.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.056216 27788 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.232.60:35871 every 8 connection(s)
21:03:19.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.059401 27712 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/data/info.pb
21:03:19.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.061944 27552 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 27712
21:03:19.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.062583 27552 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/master-2/wal/instance
21:03:19.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.066429 27789 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:19.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.080058 27789 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } has no permanent_uuid. Determining permanent_uuid...
21:03:19.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.094655 27633 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } attempt: 3
21:03:19.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.107661 27789 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } has no permanent_uuid. Determining permanent_uuid...
21:03:19.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.120985 27789 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } has no permanent_uuid. Determining permanent_uuid...
21:03:19.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.122547 27633 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218: Bootstrap starting.
21:03:19.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.130687 27633 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218: Neither blocks nor log segments found. Creating new log.
21:03:19.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.133601 27633 log.cc:826] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218: Log is configured to *not* fsync() on all Append() calls
21:03:19.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.139871 27633 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218: No bootstrap required, opened a new log
21:03:19.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.145696 27789 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8: Bootstrap starting.
21:03:19.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.155023 27789 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8: Neither blocks nor log segments found. Creating new log.
21:03:19.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.157949 27789 log.cc:826] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8: Log is configured to *not* fsync() on all Append() calls
21:03:19.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.163158 27789 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8: No bootstrap required, opened a new log
21:03:19.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.168337 27633 raft_consensus.cc:360] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } }
21:03:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.169603 27633 raft_consensus.cc:386] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:19.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.169938 27633 raft_consensus.cc:748] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6e875411d7094b09bf311957bd5d4218, State: Initialized, Role: FOLLOWER
21:03:19.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.170851 27633 consensus_queue.cc:261] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } }
21:03:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.173617 27799 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } } }
21:03:19.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.174379 27799 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [sys.catalog]: This master's current role is: FOLLOWER
21:03:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.175864 27633 sys_catalog.cc:565] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [sys.catalog]: configured and running, proceeding with master startup.
21:03:19.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.193387 27789 raft_consensus.cc:360] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } }
21:03:19.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.194416 27789 raft_consensus.cc:386] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:19.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.194782 27789 raft_consensus.cc:748] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 661fad8838a84ba4a9f47ce4a78c19e8, State: Initialized, Role: FOLLOWER
21:03:19.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.196111 27789 consensus_queue.cc:261] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } }
21:03:19.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.199599 27811 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } } }
21:03:19.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.200677 27811 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [sys.catalog]: This master's current role is: FOLLOWER
21:03:19.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.201767 27789 sys_catalog.cc:565] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [sys.catalog]: configured and running, proceeding with master startup.
21:03:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:19.229547 27810 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:03:19.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:19.230388 27810 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:03:19.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:19.238909 27743 tablet.cc:2441] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:19.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.240039 27710 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } attempt: 7
21:03:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.207108 27565 master_runner.cc:187] Error getting master registration for 127.26.232.61:37807: OK, instance_id { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" instance_seqno: 1794690196829302 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:03:19.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.252615 27565 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:03:19.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.271731 27710 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556: Bootstrap starting.
21:03:19.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.279939 27710 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556: Neither blocks nor log segments found. Creating new log.
21:03:19.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:19.281352 27824 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:03:19.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.282665 27710 log.cc:826] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556: Log is configured to *not* fsync() on all Append() calls
21:03:19.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:19.282866 27824 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:03:19.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:19.292483 27587 tablet.cc:2441] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:19.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.303907 27710 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556: No bootstrap required, opened a new log
21:03:19.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.268900 27712 master_runner.cc:187] Error getting master registration for 127.26.232.61:37807: OK, instance_id { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" instance_seqno: 1794690196829302 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:03:19.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.306818 27712 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:03:19.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.328497 27710 raft_consensus.cc:360] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } }
21:03:19.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.329388 27710 raft_consensus.cc:386] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:19.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.329694 27710 raft_consensus.cc:748] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a770a96337fb4b68b5199c7d22074556, State: Initialized, Role: FOLLOWER
21:03:19.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.330802 27710 consensus_queue.cc:261] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } }
21:03:19.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.336805 27827 sys_catalog.cc:455] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } } }
21:03:19.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.344707 27827 sys_catalog.cc:458] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [sys.catalog]: This master's current role is: FOLLOWER
21:03:19.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.339008 27710 sys_catalog.cc:565] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [sys.catalog]: configured and running, proceeding with master startup.
21:03:19.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.426923 27811 raft_consensus.cc:499] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:19.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.427492 27811 raft_consensus.cc:521] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } }
21:03:19.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.429976 27811 leader_election.cc:290] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6e875411d7094b09bf311957bd5d4218 (127.26.232.62:39275), a770a96337fb4b68b5199c7d22074556 (127.26.232.61:37807)
21:03:19.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.433821 27685 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a770a96337fb4b68b5199c7d22074556" is_pre_election: true
21:03:19.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.434705 27685 raft_consensus.cc:2490] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 661fad8838a84ba4a9f47ce4a78c19e8 in term 0.
21:03:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.436424 27739 leader_election.cc:304] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 661fad8838a84ba4a9f47ce4a78c19e8, a770a96337fb4b68b5199c7d22074556; no voters:
21:03:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.437762 27608 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6e875411d7094b09bf311957bd5d4218" is_pre_election: true
21:03:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.438570 27608 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 661fad8838a84ba4a9f47ce4a78c19e8 in term 0.
21:03:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.440330 27811 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:03:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.440670 27811 raft_consensus.cc:499] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:19.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.440985 27811 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [term 0 FOLLOWER]: Advancing to term 1
21:03:19.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:19.450608 27838 catalog_manager.cc:1609] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:03:19.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:19.451284 27838 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:03:19.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.451632 27811 raft_consensus.cc:521] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } }
21:03:19.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.464041 27635 master_runner.cc:233] Existing masters have differing terms: 0 vs 1
21:03:19.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.464543 27635 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:03:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.469915 27608 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6e875411d7094b09bf311957bd5d4218"
21:03:19.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.470494 27608 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [term 0 FOLLOWER]: Advancing to term 1
21:03:19.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.479662 27608 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 661fad8838a84ba4a9f47ce4a78c19e8 in term 1.
21:03:19.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.481159 27739 leader_election.cc:304] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 661fad8838a84ba4a9f47ce4a78c19e8, 6e875411d7094b09bf311957bd5d4218; no voters:
21:03:19.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.486999 27811 leader_election.cc:290] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [CANDIDATE]: Term 1 election: Requested vote from peers 6e875411d7094b09bf311957bd5d4218 (127.26.232.62:39275), a770a96337fb4b68b5199c7d22074556 (127.26.232.61:37807)
21:03:19.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.489583 27811 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [term 1 FOLLOWER]: Leader election won for term 1
21:03:19.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.499878 27811 raft_consensus.cc:705] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [term 1 LEADER]: Becoming Leader. State: Replica: 661fad8838a84ba4a9f47ce4a78c19e8, State: Running, Role: LEADER
21:03:19.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.501979 27811 consensus_queue.cc:238] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } }
21:03:19.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.505908 27685 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a770a96337fb4b68b5199c7d22074556"
21:03:19.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.506490 27685 raft_consensus.cc:3096] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [term 0 FOLLOWER]: Advancing to term 1
21:03:19.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.514320 27685 raft_consensus.cc:2490] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 661fad8838a84ba4a9f47ce4a78c19e8 in term 1.
21:03:19.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.531497 27840 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 661fad8838a84ba4a9f47ce4a78c19e8. Latest consensus state: current_term: 1 leader_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } } }
21:03:19.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.532370 27840 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [sys.catalog]: This master's current role is: LEADER
21:03:19.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:19.537075 27664 tablet.cc:2441] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:19.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.558085 27842 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
21:03:19.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.562096 27811 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.004s
21:03:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.566385 27843 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:03:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.577279 27842 catalog_manager.cc:1535] Initializing Kudu cluster ID...
21:03:19.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.576926 27799 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 661fad8838a84ba4a9f47ce4a78c19e8. Latest consensus state: current_term: 1 leader_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } } }
21:03:19.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.577806 27799 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [sys.catalog]: This master's current role is: FOLLOWER
21:03:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.604480 27799 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } } }
21:03:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.605244 27799 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218 [sys.catalog]: This master's current role is: FOLLOWER
21:03:19.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.618727 27827 sys_catalog.cc:455] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 661fad8838a84ba4a9f47ce4a78c19e8. Latest consensus state: current_term: 1 leader_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } } }
21:03:19.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.619449 27827 sys_catalog.cc:458] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [sys.catalog]: This master's current role is: FOLLOWER
21:03:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.638032 27811 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } } }
21:03:19.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.639415 27811 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [sys.catalog]: This master's current role is: LEADER
21:03:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.640492 27811 sys_catalog.cc:455] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } } }
21:03:19.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.641583 27811 sys_catalog.cc:458] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8 [sys.catalog]: This master's current role is: LEADER
21:03:19.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.649299 27827 sys_catalog.cc:455] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6e875411d7094b09bf311957bd5d4218" member_type: VOTER last_known_addr { host: "127.26.232.62" port: 39275 } } peers { permanent_uuid: "a770a96337fb4b68b5199c7d22074556" member_type: VOTER last_known_addr { host: "127.26.232.61" port: 37807 } } peers { permanent_uuid: "661fad8838a84ba4a9f47ce4a78c19e8" member_type: VOTER last_known_addr { host: "127.26.232.60" port: 35871 } } }
21:03:19.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.649996 27827 sys_catalog.cc:458] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556 [sys.catalog]: This master's current role is: FOLLOWER
21:03:19.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.696496 27842 catalog_manager.cc:1398] Generated new cluster ID: a77797d534564f07823fdcf46c7821e8
21:03:19.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.696812 27842 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
21:03:19.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.767453 27842 catalog_manager.cc:1421] Generated new certificate authority record
21:03:19.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.769495 27842 catalog_manager.cc:1555] Loading token signing keys...
21:03:19.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.808415 27842 catalog_manager.cc:6221] T 00000000000000000000000000000000 P 661fad8838a84ba4a9f47ce4a78c19e8: Generated new TSK 0
21:03:19.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.809978 27842 catalog_manager.cc:1565] Initializing in-progress tserver states...
21:03:19.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:19.837915 27552 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:19.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:19.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/wal
21:03:19.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/data
21:03:19.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:19.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/logs
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/data/info.pb
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:19.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.1:0
21:03:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.232.1
21:03:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.1
21:03:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:19.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:03:20.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:20.236896 27810 catalog_manager.cc:1310] Loaded cluster ID: a77797d534564f07823fdcf46c7821e8
21:03:20.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:20.238286 27810 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218: loading cluster ID for follower catalog manager: success
21:03:20.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:20.246058 27810 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218: acquiring CA information for follower catalog manager: success
21:03:20.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:20.262872 27810 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 6e875411d7094b09bf311957bd5d4218: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:03:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:20.393687 27852 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:20.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:20.394339 27852 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:20.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:20.395097 27852 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:20.457787 27838 catalog_manager.cc:1310] Loaded cluster ID: a77797d534564f07823fdcf46c7821e8
21:03:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:20.458525 27838 catalog_manager.cc:1603] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556: loading cluster ID for follower catalog manager: success
21:03:20.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:20.468900 27838 catalog_manager.cc:1625] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556: acquiring CA information for follower catalog manager: success
21:03:20.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:20.474316 27852 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:20.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:20.480546 27838 catalog_manager.cc:1653] T 00000000000000000000000000000000 P a770a96337fb4b68b5199c7d22074556: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:03:20.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:20.482386 27852 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.232.1
21:03:20.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:20.550544 27852 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:20.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/logs
21:03:20.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:20.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:20.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:20.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:20.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:20.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:20.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/data
21:03:20.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/wal
21:03:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.1:0
21:03:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/data/info.pb
21:03:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.1
21:03:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.232.1
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:20.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:20.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:20.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:20.561694 27852 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:20.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:20.564499 27852 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:20.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:20.591290 27858 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:23.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:21.990816 27857 debug-util.cc:398] Leaking SignalData structure 0x7b0800029220 after lost signal to thread 27852
21:03:23.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:20.601729 27859 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:23.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:23.678282 27862 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:23.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:23.677388 27852 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 3.075s user 0.614s sys 1.150s
21:03:23.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:23.679577 27852 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 3.079s user 0.615s sys 1.151s
21:03:23.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:23.681779 27860 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3076 milliseconds
21:03:23.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.681810 27852 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:03:23.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.683302 27852 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:23.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.697824 27852 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:23.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.699468 27852 hybrid_clock.cc:648] HybridClock initialized: now 1794690203699388 us; error 85 us; skew 500 ppm
21:03:23.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.700555 27852 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:23.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.729990 27852 webserver.cc:577] Webserver started at http://127.26.232.1:37789/ using document root <none> and password file <none>
21:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.731300 27852 fs_manager.cc:362] Metadata directory not provided
21:03:23.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.731591 27852 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:23.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.732175 27852 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:23.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.738381 27852 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/data/instance:
21:03:23.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b24efdb135145a7ae78f733255557b3"
21:03:23.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:23 on dist-test-slave-q92b"
21:03:23.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.740006 27852 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/wal/instance:
21:03:23.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b24efdb135145a7ae78f733255557b3"
21:03:23.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:23 on dist-test-slave-q92b"
21:03:23.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.755723 27852 fs_manager.cc:696] Time spent creating directory manager: real 0.015s user 0.003s sys 0.011s
21:03:23.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.768934 27870 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:23.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.770548 27852 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.008s sys 0.000s
21:03:23.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.770959 27852 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/wal
21:03:23.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4b24efdb135145a7ae78f733255557b3"
21:03:23.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:23 on dist-test-slave-q92b"
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.771417 27852 fs_report.cc:389] FS layout report
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/wal
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/wal
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/data/data
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:23.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:23.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:23.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:23.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:23.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.837322 27852 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:23.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.839195 27852 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:23.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.839706 27852 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:23.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.860224 27852 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:03:23.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.871546 27852 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:23.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.872318 27852 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.001s
21:03:23.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.872882 27852 ts_tablet_manager.cc:662] Registered 0 tablets
21:03:23.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:23.873406 27852 ts_tablet_manager.cc:641] Time spent register tablets: real 0.001s user 0.000s sys 0.000s
21:03:24.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.053800 27852 rpc_server.cc:307] RPC server started. Bound to: 127.26.232.1:41355
21:03:24.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.053889 27982 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.232.1:41355 every 8 connection(s)
21:03:24.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.057401 27852 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/data/info.pb
21:03:24.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.067466 27552 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 27852
21:03:24.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.068236 27552 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-0/wal/instance
21:03:24.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.085160 27552 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/wal
21:03:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/data
21:03:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:24.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/logs
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/data/info.pb
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.2:0
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.232.2
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.2
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:24.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:24.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:24.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:24.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:24.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:03:24.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.104420 27983 heartbeater.cc:345] Connected to a master server at 127.26.232.60:35871
21:03:24.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.104892 27983 heartbeater.cc:462] Registering TS with master...
21:03:24.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.106298 27983 heartbeater.cc:508] Master 127.26.232.60:35871 requested a full tablet report, sending...
21:03:24.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.109419 27754 ts_manager.cc:210] Registered new tserver with Master: 4b24efdb135145a7ae78f733255557b3 (127.26.232.1:41355)
21:03:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.109810 27984 heartbeater.cc:345] Connected to a master server at 127.26.232.61:37807
21:03:24.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.110137 27984 heartbeater.cc:462] Registering TS with master...
21:03:24.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.110967 27984 heartbeater.cc:508] Master 127.26.232.61:37807 requested a full tablet report, sending...
21:03:24.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.112125 27754 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.26.232.1:53179
21:03:24.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.114626 27675 ts_manager.cc:210] Registered new tserver with Master: 4b24efdb135145a7ae78f733255557b3 (127.26.232.1:41355)
21:03:24.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.118923 27985 heartbeater.cc:345] Connected to a master server at 127.26.232.62:39275
21:03:24.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.119217 27985 heartbeater.cc:462] Registering TS with master...
21:03:24.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.119674 27985 heartbeater.cc:508] Master 127.26.232.62:39275 requested a full tablet report, sending...
21:03:24.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.122123 27598 ts_manager.cc:210] Registered new tserver with Master: 4b24efdb135145a7ae78f733255557b3 (127.26.232.1:41355)
21:03:24.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:24.449934 27993 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:24.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:24.450520 27993 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:24.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:24.451123 27993 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:24.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:24.489012 27993 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:24.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:24.490125 27993 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.232.2
21:03:24.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.527057 27993 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:24.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/logs
21:03:24.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:24.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/data
21:03:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/wal
21:03:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:24.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.2:0
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/data/info.pb
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.2
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:24.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.232.2
21:03:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.528762 27993 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:24.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:24.531157 27993 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:24.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:24.544497 27999 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:25.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.117548 27983 heartbeater.cc:500] Master 127.26.232.60:35871 was elected leader, sending a full tablet report...
21:03:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:24.545888 28000 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:25.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:24.548748 28002 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:25.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:25.801923 28001 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1251 milliseconds
21:03:25.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.802024 27993 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:03:25.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.803339 27993 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:25.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.805868 27993 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.807271 27993 hybrid_clock.cc:648] HybridClock initialized: now 1794690205807221 us; error 65 us; skew 500 ppm
21:03:25.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.808143 27993 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:25.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.814769 27993 webserver.cc:577] Webserver started at http://127.26.232.2:35347/ using document root <none> and password file <none>
21:03:25.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.815773 27993 fs_manager.cc:362] Metadata directory not provided
21:03:25.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.816020 27993 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:25.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.816591 27993 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.821251 27993 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/data/instance:
21:03:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be501b7277bd494b91d4750b6b1ed2dd"
21:03:25.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:25 on dist-test-slave-q92b"
21:03:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.822501 27993 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/wal/instance:
21:03:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be501b7277bd494b91d4750b6b1ed2dd"
21:03:25.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:25 on dist-test-slave-q92b"
21:03:25.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.829890 27993 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.000s
21:03:25.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.835791 28010 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:25.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.837116 27993 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
21:03:25.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.837560 27993 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/wal
21:03:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "be501b7277bd494b91d4750b6b1ed2dd"
21:03:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:25 on dist-test-slave-q92b"
21:03:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.837935 27993 fs_report.cc:389] FS layout report
21:03:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/wal
21:03:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/wal
21:03:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/data/data
21:03:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:25.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:25.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:25.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.895639 27993 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:25.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.897338 27993 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:25.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.897877 27993 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:25.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.900669 27993 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:03:25.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.906847 27993 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:25.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.907538 27993 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.000s
21:03:25.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.907881 27993 ts_tablet_manager.cc:662] Registered 0 tablets
21:03:25.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:25.908042 27993 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:03:26.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.067694 27993 rpc_server.cc:307] RPC server started. Bound to: 127.26.232.2:40037
21:03:26.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.067803 28122 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.232.2:40037 every 8 connection(s)
21:03:26.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.071894 27993 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/data/info.pb
21:03:26.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.080552 27552 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 27993
21:03:26.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.081099 27552 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-1/wal/instance
21:03:26.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.099025 27552 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/wal
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/data
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/logs
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/data/info.pb
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:26.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.3:0
21:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.232.3
21:03:26.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.3
21:03:26.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:26.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:26.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:26.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:26.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:26.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:03:26.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.130080 28123 heartbeater.cc:345] Connected to a master server at 127.26.232.60:35871
21:03:26.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.130625 28123 heartbeater.cc:462] Registering TS with master...
21:03:26.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.131631 28124 heartbeater.cc:345] Connected to a master server at 127.26.232.61:37807
21:03:26.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.132020 28124 heartbeater.cc:462] Registering TS with master...
21:03:26.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.132120 28123 heartbeater.cc:508] Master 127.26.232.60:35871 requested a full tablet report, sending...
21:03:26.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.132961 28124 heartbeater.cc:508] Master 127.26.232.61:37807 requested a full tablet report, sending...
21:03:26.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.133285 28125 heartbeater.cc:345] Connected to a master server at 127.26.232.62:39275
21:03:26.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.133620 28125 heartbeater.cc:462] Registering TS with master...
21:03:26.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.134924 28125 heartbeater.cc:508] Master 127.26.232.62:39275 requested a full tablet report, sending...
21:03:26.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.135671 27754 ts_manager.cc:210] Registered new tserver with Master: be501b7277bd494b91d4750b6b1ed2dd (127.26.232.2:40037)
21:03:26.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.136267 27675 ts_manager.cc:210] Registered new tserver with Master: be501b7277bd494b91d4750b6b1ed2dd (127.26.232.2:40037)
21:03:26.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.137509 27754 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.26.232.2:37593
21:03:26.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.139919 27598 ts_manager.cc:210] Registered new tserver with Master: be501b7277bd494b91d4750b6b1ed2dd (127.26.232.2:40037)
21:03:26.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:26.455420 28130 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:26.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:26.456017 28130 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:26.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:26.456480 28130 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:26.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:26.491469 28130 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:26.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:26.492312 28130 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.26.232.3
21:03:26.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.530540 28130 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:26.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/logs
21:03:26.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:26.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:26.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:26.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.26.232.20:35655
21:03:26.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:26.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:26.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/data
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/wal
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.26.232.3:0
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/data/info.pb
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.26.232.3
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.26.232.62:39275,127.26.232.61:37807,127.26.232.60:35871
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:26.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.26.232.3
21:03:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:26.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.532019 28130 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:26.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:26.533903 28130 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:26.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:26.548207 28139 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:27.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:27.143043 28123 heartbeater.cc:500] Master 127.26.232.60:35871 was elected leader, sending a full tablet report...
21:03:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:27.953608 28138 debug-util.cc:398] Leaking SignalData structure 0x7b0800029220 after lost signal to thread 28130
21:03:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:27.989856 28138 kernel_stack_watchdog.cc:198] Thread 28130 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 403ms:
21:03:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:03:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:03:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:03:27.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:03:27.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:26.548614 28140 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:27.991811 28130 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.443s user 0.495s sys 0.920s
21:03:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:27.992285 28141 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1443 milliseconds
21:03:27.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:27.992467 28130 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.445s user 0.495s sys 0.920s
21:03:27.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:27.992991 28142 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:27.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:27.993029 28130 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:03:27.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:27.997715 28130 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:28.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:27.999996 28130 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:28.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.001397 28130 hybrid_clock.cc:648] HybridClock initialized: now 1794690208001355 us; error 39 us; skew 500 ppm
21:03:28.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.002326 28130 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:28.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.008931 28130 webserver.cc:577] Webserver started at http://127.26.232.3:34049/ using document root <none> and password file <none>
21:03:28.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.009996 28130 fs_manager.cc:362] Metadata directory not provided
21:03:28.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.010280 28130 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:28.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.010864 28130 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:28.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.015483 28130 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/data/instance:
21:03:28.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a5d32116a7f46f5b83cef1eb161c238"
21:03:28.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:28 on dist-test-slave-q92b"
21:03:28.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.016799 28130 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/wal/instance:
21:03:28.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a5d32116a7f46f5b83cef1eb161c238"
21:03:28.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:28 on dist-test-slave-q92b"
21:03:28.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.024789 28130 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.005s
21:03:28.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.030627 28150 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.031801 28130 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
21:03:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.032192 28130 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/wal
21:03:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4a5d32116a7f46f5b83cef1eb161c238"
21:03:28.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:28 on dist-test-slave-q92b"
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.032579 28130 fs_report.cc:389] FS layout report
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/wal
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/wal
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/data/data
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:28.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:28.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:28.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:28.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:28.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:28.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.107963 28130 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:28.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.109665 28130 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:28.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.110249 28130 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:28.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.113073 28130 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:03:28.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.117796 28130 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:28.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.118060 28130 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:03:28.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.118433 28130 ts_tablet_manager.cc:662] Registered 0 tablets
21:03:28.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.118665 28130 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.275803 28130 rpc_server.cc:307] RPC server started. Bound to: 127.26.232.3:36609
21:03:28.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.275883 28262 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.26.232.3:36609 every 8 connection(s)
21:03:28.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.278640 28130 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/data/info.pb
21:03:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.283806 27552 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 28130
21:03:28.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.284669 27552 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster7753065665411095180/ts-2/wal/instance
21:03:28.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.322403 28265 heartbeater.cc:345] Connected to a master server at 127.26.232.62:39275
21:03:28.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.323338 28265 heartbeater.cc:462] Registering TS with master...
21:03:28.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.324223 28263 heartbeater.cc:345] Connected to a master server at 127.26.232.60:35871
21:03:28.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.324575 28263 heartbeater.cc:462] Registering TS with master...
21:03:28.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.324729 28265 heartbeater.cc:508] Master 127.26.232.62:39275 requested a full tablet report, sending...
21:03:28.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.325459 28264 heartbeater.cc:345] Connected to a master server at 127.26.232.61:37807
21:03:28.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.325800 28264 heartbeater.cc:462] Registering TS with master...
21:03:28.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.325754 28263 heartbeater.cc:508] Master 127.26.232.60:35871 requested a full tablet report, sending...
21:03:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.327157 28264 heartbeater.cc:508] Master 127.26.232.61:37807 requested a full tablet report, sending...
21:03:28.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.328558 27754 ts_manager.cc:210] Registered new tserver with Master: 4a5d32116a7f46f5b83cef1eb161c238 (127.26.232.3:36609)
21:03:28.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.329039 27598 ts_manager.cc:210] Registered new tserver with Master: 4a5d32116a7f46f5b83cef1eb161c238 (127.26.232.3:36609)
21:03:28.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.330001 27674 ts_manager.cc:210] Registered new tserver with Master: 4a5d32116a7f46f5b83cef1eb161c238 (127.26.232.3:36609)
21:03:28.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.330250 27754 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.26.232.3:57179
21:03:28.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:28.339936 27552 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
21:03:28.352 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:03:28.353 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:03:28.365 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.26.232.62"
port: 39275
}
bound_http_address {
host: "127.26.232.62"
port: 41899
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.26.232.61"
port: 37807
}
bound_http_address {
host: "127.26.232.61"
port: 35973
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.26.232.60"
port: 35871
}
bound_http_address {
host: "127.26.232.60"
port: 35705
}
}
}
21:03:28.369 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:03:28.373 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.26.232.1"
port: 41355
}
bound_http_address {
host: "127.26.232.1"
port: 37789
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.26.232.2"
port: 40037
}
bound_http_address {
host: "127.26.232.2"
port: 35347
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.26.232.3"
port: 36609
}
bound_http_address {
host: "127.26.232.3"
port: 34049
}
}
}
21:03:28.373 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:03:28.564 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
21:03:28.622 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:03:28.713 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskerlxO1/test-tmp
21:03:28.713 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:03:28.714 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskerlxO1/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:03:28.734 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
21:03:29.090 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:29.090 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:03:29.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:29.094039 28275 mini_oidc.cc:150] Starting JWKS server
21:03:29.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:29.097718 28275 webserver.cc:577] Webserver started at http://127.0.0.1:37809/ using document root <none> and password file <none>
21:03:29.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:29.098260 28275 mini_oidc.cc:186] Starting OIDC Discovery server
21:03:29.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:29.100420 28275 webserver.cc:577] Webserver started at http://127.0.0.1:46481/ using document root <none> and password file <none>
21:03:29.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
21:03:29.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/principal' for realm 'KRBTEST.COM',
21:03:29.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
21:03:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:29 dist-test-slave-q92b krb5kdc[28287](info): setting up network...
21:03:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
21:03:29.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:29 dist-test-slave-q92b krb5kdc[28287](info): set up 2 sockets
21:03:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:29 dist-test-slave-q92b krb5kdc[28287](info): commencing operation
21:03:29.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
21:03:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:29.333484 28263 heartbeater.cc:500] Master 127.26.232.60:35871 was elected leader, sending a full tablet report...
21:03:31.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.248903 28275 mini_kdc.cc:121] Time spent starting KDC: real 2.112s user 0.009s sys 0.012s
21:03:31.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
21:03:31.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:03:31.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
21:03:31.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
21:03:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:03:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
21:03:31.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
21:03:31.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:03:31.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
21:03:31.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:03:31.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/test-user.keytab.
21:03:31.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/test-user.keytab.
21:03:31.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:31 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690211, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:31.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:03:31.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-14T21:03:31Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:03:31.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-14T21:03:31Z Disabled control of system clock
21:03:31.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.156.254@KRBTEST.COM; defaulting to no policy
21:03:31.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:31.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.156.254@KRBTEST.COM" created.
21:03:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:31.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.156.254@KRBTEST.COM; defaulting to no policy
21:03:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:31.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.156.254@KRBTEST.COM" created.
21:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:31.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:31.490511 28275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:31.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/logs
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data/info.pb
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.254:34367
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.254
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.254
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:03:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:03:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.816959 28304 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:31.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.817651 28304 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:31.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.818392 28304 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:31.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.853413 28304 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
21:03:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.853705 28304 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:03:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.854030 28304 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.854269 28304 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.854506 28304 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:03:31.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.854777 28304 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:31.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.854959 28304 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37809/jwks.json
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:31.896032 28304 master_runner.cc:387] Master server non-default flags:
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/logs
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.254
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.254:34367
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data/info.pb
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.254
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:31.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:31.897449 28304 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:31.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:31.898449 28304 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:31.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.911871 28310 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:31.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.929749 28119 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad1c0 after lost signal to thread 27994
21:03:31.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.931465 28119 debug-util.cc:398] Leaking SignalData structure 0x7b08000b88e0 after lost signal to thread 28122
21:03:33.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:33.555303 27706 debug-util.cc:398] Leaking SignalData structure 0x7b0800099a20 after lost signal to thread 27639
21:03:33.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:33.556248 27706 debug-util.cc:398] Leaking SignalData structure 0x7b0800070040 after lost signal to thread 27709
21:03:34.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:33.312772 28309 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 28304
21:03:34.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:34.005602 28309 kernel_stack_watchdog.cc:198] Thread 28304 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 398ms:
21:03:34.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:03:34.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:03:34.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:03:34.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:03:34.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:34.011209 28304 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 2.099s user 0.697s sys 1.251s
21:03:34.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:34.012315 28304 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 2.100s user 0.697s sys 1.252s
21:03:34.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:34.012405 28313 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:34.026721 28312 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2112 milliseconds
21:03:34.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.027446 28304 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:03:34.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:31.912144 28311 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.035049 28304 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:34.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.038939 28304 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:34.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.040761 28304 hybrid_clock.cc:648] HybridClock initialized: now 1794690214040681 us; error 88 us; skew 500 ppm
21:03:34.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.044931 28304 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:34.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:34 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690214, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.053460 28304 init.cc:377] Logged in from keytab as kudu/127.27.156.254@KRBTEST.COM (short username kudu)
21:03:34.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.066821 28304 webserver.cc:577] Webserver started at http://127.27.156.254:39973/ using document root <none> and password file <none>
21:03:34.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.068357 28304 fs_manager.cc:362] Metadata directory not provided
21:03:34.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.068821 28304 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:34.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.069536 28304 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.075836 28304 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data/instance:
21:03:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef2ace3461a84b6a82ef8b5c267c7bff"
21:03:34.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:34 on dist-test-slave-q92b"
21:03:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.077553 28304 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal/instance:
21:03:34.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef2ace3461a84b6a82ef8b5c267c7bff"
21:03:34.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:34 on dist-test-slave-q92b"
21:03:34.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.107965 28304 fs_manager.cc:696] Time spent creating directory manager: real 0.029s user 0.007s sys 0.019s
21:03:34.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.133803 28323 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:34.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.136605 28304 fs_manager.cc:730] Time spent opening block manager: real 0.024s user 0.009s sys 0.014s
21:03:34.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.137436 28304 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal
21:03:34.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef2ace3461a84b6a82ef8b5c267c7bff"
21:03:34.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:34 on dist-test-slave-q92b"
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.138255 28304 fs_report.cc:389] FS layout report
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data/data
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:34.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:34.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.211793 28304 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:34.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.226613 28304 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:34.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.227532 28304 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:34.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.377821 28304 rpc_server.cc:307] RPC server started. Bound to: 127.27.156.254:34367
21:03:34.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.382431 28304 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data/info.pb
21:03:34.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.377911 28374 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.156.254:34367 every 8 connection(s)
21:03:34.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.388662 28275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 28304
21:03:34.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.389174 28275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal/instance
21:03:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.409082 28375 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:34.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.156.253@KRBTEST.COM; defaulting to no policy
21:03:34.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:34.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.156.253@KRBTEST.COM" created.
21:03:34.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.437825 28375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } has no permanent_uuid. Determining permanent_uuid...
21:03:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:34.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:34.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:34 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690214, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.254@KRBTEST.COM for kudu/127.27.156.254@KRBTEST.COM
21:03:34.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.156.253@KRBTEST.COM; defaulting to no policy
21:03:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:34.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.156.253@KRBTEST.COM" created.
21:03:34.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.498994 28375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } has no permanent_uuid. Determining permanent_uuid...
21:03:34.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:34.502575 28324 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:34.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:34.508507 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.253:44373: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:34.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:34.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:34.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.514153 28275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:34.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:34.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal
21:03:34.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data
21:03:34.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:34.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:34.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:34.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/logs
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data/info.pb
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.253:44373
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.253
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.253
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:34.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:03:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:03:34.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.558079 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } attempt: 1
21:03:34.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:34.564790 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.253:44373: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:34.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.633270 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } attempt: 2
21:03:34.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:34.640092 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.253:44373: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:34.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.731529 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } attempt: 3
21:03:34.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:34.738369 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.253:44373: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:34.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:34.881807 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } attempt: 4
21:03:34.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:34.888620 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.253:44373: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:35.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.049142 28385 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:35.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.050069 28385 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:35.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.052307 28385 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.125316 28385 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
21:03:35.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.125695 28385 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:03:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.126260 28385 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:35.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.126670 28385 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:35.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.127074 28385 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:03:35.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.127482 28385 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:35.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.127774 28385 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37809/jwks.json
21:03:35.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:35.188062 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } attempt: 5
21:03:35.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.194134 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.253:44373: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:35.211710 28385 master_runner.cc:387] Master server non-default flags:
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/logs
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.253
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:35.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.253:44373
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data/info.pb
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.253
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:35.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:35.213524 28385 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:35.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:35.214939 28385 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:35.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.231487 28392 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:35.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:35.741566 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } attempt: 6
21:03:35.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.753159 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.253:44373: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:36.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.234243 28393 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:36.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:35.235898 28395 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:36.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:36.597074 28394 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:03:36.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.597225 28385 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:03:36.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.601873 28385 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:36.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.604586 28385 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:36.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.605947 28385 hybrid_clock.cc:648] HybridClock initialized: now 1794690216605917 us; error 80 us; skew 500 ppm
21:03:36.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.606796 28385 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:36.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:36 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690216, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:36.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.613576 28385 init.cc:377] Logged in from keytab as kudu/127.27.156.253@KRBTEST.COM (short username kudu)
21:03:36.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.620225 28385 webserver.cc:577] Webserver started at http://127.27.156.253:39629/ using document root <none> and password file <none>
21:03:36.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.621393 28385 fs_manager.cc:362] Metadata directory not provided
21:03:36.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.621732 28385 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:36.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.622344 28385 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:36.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.626835 28385 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data/instance:
21:03:36.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b1a43974a60e4addb0042d681f0649e4"
21:03:36.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:36 on dist-test-slave-q92b"
21:03:36.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.628072 28385 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal/instance:
21:03:36.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b1a43974a60e4addb0042d681f0649e4"
21:03:36.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:36 on dist-test-slave-q92b"
21:03:36.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.636215 28385 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
21:03:36.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.642869 28406 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:36.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.644061 28385 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
21:03:36.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.644457 28385 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal
21:03:36.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b1a43974a60e4addb0042d681f0649e4"
21:03:36.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:36 on dist-test-slave-q92b"
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.644824 28385 fs_report.cc:389] FS layout report
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data/data
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:36.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.695385 28385 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:36.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.701120 28385 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:36.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.701813 28385 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.785717 28385 rpc_server.cc:307] RPC server started. Bound to: 127.27.156.253:44373
21:03:36.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.785832 28457 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.156.253:44373 every 8 connection(s)
21:03:36.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.788746 28385 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data/info.pb
21:03:36.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.795807 28458 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:36.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.796168 28275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 28385
21:03:36.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.796537 28275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal/instance
21:03:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.813635 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } attempt: 7
21:03:36.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.156.252@KRBTEST.COM; defaulting to no policy
21:03:36.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.811343 28458 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } has no permanent_uuid. Determining permanent_uuid...
21:03:36.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:36.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.156.252@KRBTEST.COM" created.
21:03:36.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:36 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690214, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.254@KRBTEST.COM for kudu/127.27.156.253@KRBTEST.COM
21:03:36.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:36 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690216, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.253@KRBTEST.COM for kudu/127.27.156.254@KRBTEST.COM
21:03:36.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:36.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:36.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:36.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.860066 28458 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } has no permanent_uuid. Determining permanent_uuid...
21:03:36.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.861568 28375 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } has no permanent_uuid. Determining permanent_uuid...
21:03:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.156.252@KRBTEST.COM; defaulting to no policy
21:03:36.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:36.869983 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:36.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:36.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.156.252@KRBTEST.COM" created.
21:03:36.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:36 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690216, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.253@KRBTEST.COM for kudu/127.27.156.253@KRBTEST.COM
21:03:36.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.886255 28458 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } has no permanent_uuid. Determining permanent_uuid...
21:03:36.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:36.889015 28408 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:36.894476 28458 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:36.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:36.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:36.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.897859 28275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/logs
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data/info.pb
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.252:44355
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.252
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.252
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:03:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:03:36.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.928562 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 1
21:03:36.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:36.932821 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:36.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:36.944001 28458 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 1
21:03:36.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:36.948732 28458 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:37.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:37.014204 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 2
21:03:37.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:37.017242 28458 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 2
21:03:37.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.019912 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:37.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.022451 28458 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:37.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:37.105336 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 3
21:03:37.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.110106 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:37.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:37.113865 28458 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 3
21:03:37.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.118991 28458 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:37.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:37.250567 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 4
21:03:37.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.255911 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:37.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:37.262645 28458 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 4
21:03:37.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.264671 28470 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:37.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.265568 28470 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:37.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.266413 28470 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:37.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.267342 28458 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.309263 28470 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
21:03:37.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.309674 28470 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:03:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.310029 28470 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.310335 28470 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.310590 28470 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:03:37.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.310806 28470 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:37.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.310962 28470 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37809/jwks.json
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:37.349917 28470 master_runner.cc:387] Master server non-default flags:
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/logs
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:03:37.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.252
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.252:44355
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data/info.pb
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.252
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:37.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:37.351300 28470 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:37.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:37.352324 28470 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:37.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.369011 28477 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:37.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:37.539389 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 5
21:03:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.547083 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:37.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:37.566802 28458 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 5
21:03:37.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.574239 28458 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:38.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.099717 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 6
21:03:38.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.121850 28458 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 6
21:03:38.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:38.139593 28375 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:38.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:38.141294 28458 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.27.156.252:44355: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:38.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:37.372351 28478 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:38.719735 28470 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 1.350s user 0.447s sys 0.889s
21:03:38.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:38.720125 28470 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 1.350s user 0.447s sys 0.889s
21:03:38.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.722200 28470 server_base.cc:1061] running on GCE node
21:03:38.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:38.724105 28486 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:38.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.725745 28470 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:38.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.728960 28470 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:38.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.732550 28470 hybrid_clock.cc:648] HybridClock initialized: now 1794690218732498 us; error 51 us; skew 500 ppm
21:03:38.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.733857 28470 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:38.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:38 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690218, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:38.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.741287 28470 init.cc:377] Logged in from keytab as kudu/127.27.156.252@KRBTEST.COM (short username kudu)
21:03:38.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.748607 28470 webserver.cc:577] Webserver started at http://127.27.156.252:44327/ using document root <none> and password file <none>
21:03:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.749764 28470 fs_manager.cc:362] Metadata directory not provided
21:03:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.750033 28470 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:38.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.750622 28470 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:38.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.755473 28470 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data/instance:
21:03:38.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "78a99a0f6f524309b49ee7391870993d"
21:03:38.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:38 on dist-test-slave-q92b"
21:03:38.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.756819 28470 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal/instance:
21:03:38.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "78a99a0f6f524309b49ee7391870993d"
21:03:38.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:38 on dist-test-slave-q92b"
21:03:38.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.765511 28470 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.001s sys 0.005s
21:03:38.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.771488 28492 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.772819 28470 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
21:03:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.773247 28470 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal
21:03:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "78a99a0f6f524309b49ee7391870993d"
21:03:38.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:38 on dist-test-slave-q92b"
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.773600 28470 fs_report.cc:389] FS layout report
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data/data
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:38.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:38.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:38.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.849550 28470 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:38.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.855410 28470 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:38.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.856024 28470 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:38.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.945904 28470 rpc_server.cc:307] RPC server started. Bound to: 127.27.156.252:44355
21:03:38.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.945962 28543 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.156.252:44355 every 8 connection(s)
21:03:38.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.948973 28470 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data/info.pb
21:03:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.955019 28275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 28470
21:03:38.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.955511 28275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal/instance
21:03:38.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.955778 28544 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:03:38.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:38.977169 28544 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } has no permanent_uuid. Determining permanent_uuid...
21:03:38.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:38 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690211, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.156.254@KRBTEST.COM
21:03:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:39 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690218, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.252@KRBTEST.COM for kudu/127.27.156.254@KRBTEST.COM
21:03:39.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.020151 28544 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } has no permanent_uuid. Determining permanent_uuid...
21:03:39.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:39 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690211, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.156.253@KRBTEST.COM
21:03:39.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:39 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690218, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.252@KRBTEST.COM for kudu/127.27.156.253@KRBTEST.COM
21:03:39.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.043372 28544 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } has no permanent_uuid. Determining permanent_uuid...
21:03:39.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:39 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690211, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.27.156.252@KRBTEST.COM
21:03:39.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:39 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690218, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.252@KRBTEST.COM for kudu/127.27.156.252@KRBTEST.COM
21:03:39.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.077050 28544 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: Bootstrap starting.
21:03:39.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.085209 28544 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: Neither blocks nor log segments found. Creating new log.
21:03:39.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.087404 28544 log.cc:826] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: Log is configured to *not* fsync() on all Append() calls
21:03:39.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.093451 28544 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: No bootstrap required, opened a new log
21:03:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.114564 28544 raft_consensus.cc:360] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.115163 28544 raft_consensus.cc:386] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.115353 28544 raft_consensus.cc:748] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 78a99a0f6f524309b49ee7391870993d, State: Initialized, Role: FOLLOWER
21:03:39.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.116079 28544 consensus_queue.cc:261] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:39.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.119076 28556 sys_catalog.cc:455] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:39.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.119735 28556 sys_catalog.cc:458] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.121119 28544 sys_catalog.cc:565] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: configured and running, proceeding with master startup.
21:03:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:39.155066 28567 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:03:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:39.155426 28567 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:03:39.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.173168 28375 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 7
21:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.147760 28470 master_runner.cc:187] Error getting master registration for 127.27.156.254:34367: OK, instance_id { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" instance_seqno: 1794690214328586 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:03:39.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.180260 28470 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:03:39.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:39 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690214, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.254@KRBTEST.COM for kudu/127.27.156.252@KRBTEST.COM
21:03:39.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.201829 28458 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } attempt: 7
21:03:39.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.210402 28375 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: Bootstrap starting.
21:03:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.220901 28375 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: Neither blocks nor log segments found. Creating new log.
21:03:39.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.223910 28375 log.cc:826] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: Log is configured to *not* fsync() on all Append() calls
21:03:39.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.230487 28375 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: No bootstrap required, opened a new log
21:03:39.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:39 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690216, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.253@KRBTEST.COM for kudu/127.27.156.252@KRBTEST.COM
21:03:39.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.257499 28375 raft_consensus.cc:360] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.258687 28375 raft_consensus.cc:386] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:39.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.259156 28375 raft_consensus.cc:748] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef2ace3461a84b6a82ef8b5c267c7bff, State: Initialized, Role: FOLLOWER
21:03:39.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.260360 28375 consensus_queue.cc:261] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:39.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.269840 28572 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:39.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.273977 28572 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.273633 28375 sys_catalog.cc:565] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: configured and running, proceeding with master startup.
21:03:39.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.283636 28458 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: Bootstrap starting.
21:03:39.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.288522 28556 raft_consensus.cc:499] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:39.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.289650 28556 raft_consensus.cc:521] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:39.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.292948 28556 leader_election.cc:290] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ef2ace3461a84b6a82ef8b5c267c7bff (127.27.156.254:34367), b1a43974a60e4addb0042d681f0649e4 (127.27.156.253:44373)
21:03:39.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.294508 28350 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "78a99a0f6f524309b49ee7391870993d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" is_pre_election: true
21:03:39.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.295349 28350 raft_consensus.cc:2490] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 78a99a0f6f524309b49ee7391870993d in term 0.
21:03:39.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.296885 28494 leader_election.cc:304] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 78a99a0f6f524309b49ee7391870993d, ef2ace3461a84b6a82ef8b5c267c7bff; no voters:
21:03:39.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.297847 28556 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 0 FOLLOWER]: Leader pre-election won for term 1
21:03:39.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.298300 28556 raft_consensus.cc:499] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:39.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.298804 28556 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 0 FOLLOWER]: Advancing to term 1
21:03:39.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.334583 28556 raft_consensus.cc:521] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:39.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.348896 28350 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "78a99a0f6f524309b49ee7391870993d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff"
21:03:39.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.349546 28350 raft_consensus.cc:3096] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 0 FOLLOWER]: Advancing to term 1
21:03:39.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:39.363493 28498 tablet.cc:2441] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:39.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.369668 28350 raft_consensus.cc:2490] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 78a99a0f6f524309b49ee7391870993d in term 1.
21:03:39.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.377859 28494 leader_election.cc:304] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 78a99a0f6f524309b49ee7391870993d, ef2ace3461a84b6a82ef8b5c267c7bff; no voters:
21:03:39.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.380059 28556 leader_election.cc:290] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [CANDIDATE]: Term 1 election: Requested vote from peers ef2ace3461a84b6a82ef8b5c267c7bff (127.27.156.254:34367), b1a43974a60e4addb0042d681f0649e4 (127.27.156.253:44373)
21:03:39.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.387780 28576 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 1 FOLLOWER]: Leader election won for term 1
21:03:39.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.387738 28458 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: Neither blocks nor log segments found. Creating new log.
21:03:39.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.402158 28576 raft_consensus.cc:705] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 1 LEADER]: Becoming Leader. State: Replica: 78a99a0f6f524309b49ee7391870993d, State: Running, Role: LEADER
21:03:39.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.403277 28576 consensus_queue.cc:238] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:39.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.408802 28458 log.cc:826] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: Log is configured to *not* fsync() on all Append() calls
21:03:39.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.440322 28458 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: No bootstrap required, opened a new log
21:03:39.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.456319 28576 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [LEADER]: Connected to new peer: Peer: permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
21:03:39.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.457875 28556 sys_catalog.cc:455] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 78a99a0f6f524309b49ee7391870993d. Latest consensus state: current_term: 1 leader_uuid: "78a99a0f6f524309b49ee7391870993d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:39.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.464977 28556 sys_catalog.cc:458] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: This master's current role is: LEADER
21:03:39.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.480589 28572 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: SysCatalogTable state changed. Reason: New leader 78a99a0f6f524309b49ee7391870993d. Latest consensus state: current_term: 1 leader_uuid: "78a99a0f6f524309b49ee7391870993d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:39.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.482303 28458 raft_consensus.cc:360] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:39.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.483582 28458 raft_consensus.cc:386] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:03:39.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.483901 28458 raft_consensus.cc:748] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b1a43974a60e4addb0042d681f0649e4, State: Initialized, Role: FOLLOWER
21:03:39.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.483482 28572 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.484925 28458 consensus_queue.cc:261] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:39.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.493497 28587 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:39.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.494313 28587 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.497125 28458 sys_catalog.cc:565] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: configured and running, proceeding with master startup.
21:03:39.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.497377 28433 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "78a99a0f6f524309b49ee7391870993d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b1a43974a60e4addb0042d681f0649e4" is_pre_election: true
21:03:39.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.498812 28433 raft_consensus.cc:2490] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 78a99a0f6f524309b49ee7391870993d in term 0.
21:03:39.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.501550 28432 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "78a99a0f6f524309b49ee7391870993d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b1a43974a60e4addb0042d681f0649e4"
21:03:39.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.502815 28432 raft_consensus.cc:3096] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [term 0 FOLLOWER]: Advancing to term 1
21:03:39.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:39.517407 28329 tablet.cc:2441] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:03:39.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.523494 28586 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
21:03:39.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.517120 28432 raft_consensus.cc:2490] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 78a99a0f6f524309b49ee7391870993d in term 1.
21:03:39.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.541204 28576 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [LEADER]: Connected to new peer: Peer: permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:03:39.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.564930 28572 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "78a99a0f6f524309b49ee7391870993d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:39.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.566082 28572 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.572540 28577 sys_catalog.cc:455] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "78a99a0f6f524309b49ee7391870993d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:39.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.573822 28577 sys_catalog.cc:458] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: This master's current role is: LEADER
21:03:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.590072 28587 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 78a99a0f6f524309b49ee7391870993d. Latest consensus state: current_term: 1 leader_uuid: "78a99a0f6f524309b49ee7391870993d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:39.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.590801 28587 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.594463 28556 sys_catalog.cc:455] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "78a99a0f6f524309b49ee7391870993d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:39.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.595167 28556 sys_catalog.cc:458] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: This master's current role is: LEADER
21:03:39.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.597808 28586 catalog_manager.cc:1535] Initializing Kudu cluster ID...
21:03:39.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.617913 28595 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:03:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.632766 28587 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "78a99a0f6f524309b49ee7391870993d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:39.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.633489 28587 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: This master's current role is: FOLLOWER
21:03:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.657943 28586 catalog_manager.cc:1398] Generated new cluster ID: 7c1de3e32ddb4085aac5187ca386de05
21:03:39.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.658294 28586 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
21:03:39.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.647179 28304 master_runner.cc:187] Error getting master registration for 127.27.156.253:44373: OK, instance_id { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" instance_seqno: 1794690216762982 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:03:39.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.682687 28304 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:03:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.711123 28608 catalog_manager.cc:1310] Loaded cluster ID: 7c1de3e32ddb4085aac5187ca386de05
21:03:39.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.711436 28608 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: loading cluster ID for follower catalog manager: success
21:03:39.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:39.715899 28608 catalog_manager.cc:1627] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: acquiring CA information for follower catalog manager: Not found: root CA entry not found
21:03:39.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:39.716246 28608 catalog_manager.cc:898] Not found: root CA entry not found: failed to prepare follower catalog manager, will retry
21:03:39.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.739137 28586 catalog_manager.cc:1421] Generated new certificate authority record
21:03:39.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.741242 28586 catalog_manager.cc:1555] Loading token signing keys...
21:03:39.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.779436 28586 catalog_manager.cc:6221] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: Generated new TSK 0
21:03:39.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.780624 28586 catalog_manager.cc:1565] Initializing in-progress tserver states...
21:03:39.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.156.193@KRBTEST.COM; defaulting to no policy
21:03:39.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:39.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.156.193@KRBTEST.COM" created.
21:03:39.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:39.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:39.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:39.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.156.193@KRBTEST.COM; defaulting to no policy
21:03:39.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:39.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.156.193@KRBTEST.COM" created.
21:03:39.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:39.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:39.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:39.910624 28275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/wal
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/data
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/logs
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/data/info.pb
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:39.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.193:0
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.156.193
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.193
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.193
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:39.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:03:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:40.443590 28614 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:40.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:40.444285 28614 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:40.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:40.445326 28614 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:40.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:40.509864 28614 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:40.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:40.511620 28614 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.156.193
21:03:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:40.586035 27785 debug-util.cc:398] Leaking SignalData structure 0x7b0800070060 after lost signal to thread 27717
21:03:40.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:40.587014 27785 debug-util.cc:398] Leaking SignalData structure 0x7b080008cd40 after lost signal to thread 27788
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:40.590685 28614 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/logs
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/data
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/wal
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.193
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.193:0
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/data/info.pb
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.193
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.156.193
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:40.592459 28614 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:40.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:40.593854 28614 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:40.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:40.628840 28621 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:40.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:40.646431 28595 catalog_manager.cc:1310] Loaded cluster ID: 7c1de3e32ddb4085aac5187ca386de05
21:03:40.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:40.647166 28595 catalog_manager.cc:1603] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: loading cluster ID for follower catalog manager: success
21:03:40.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:40.654652 28595 catalog_manager.cc:1625] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: acquiring CA information for follower catalog manager: success
21:03:40.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:40.661406 28595 catalog_manager.cc:1653] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:03:40.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:40.725991 28608 catalog_manager.cc:1625] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: acquiring CA information for follower catalog manager: success
21:03:40.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:40.745376 28608 catalog_manager.cc:1653] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:03:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:40.643901 28614 server_base.cc:1061] running on GCE node
21:03:43.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:40.647176 28626 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:43.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:40.643290 28623 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:43.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.203457 28614 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:43.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.214046 28614 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:43.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.215663 28614 hybrid_clock.cc:648] HybridClock initialized: now 1794690223215626 us; error 47 us; skew 500 ppm
21:03:43.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.216807 28614 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:43.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:43 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690223, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:43.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.225415 28614 init.cc:377] Logged in from keytab as kudu/127.27.156.193@KRBTEST.COM (short username kudu)
21:03:43.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.232888 28614 webserver.cc:577] Webserver started at http://127.27.156.193:33359/ using document root <none> and password file <none>
21:03:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.234153 28614 fs_manager.cc:362] Metadata directory not provided
21:03:43.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.234421 28614 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:43.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.234979 28614 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.241159 28614 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/data/instance:
21:03:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db9a128d9d4046bf915e0e521512c1d6"
21:03:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:43 on dist-test-slave-q92b"
21:03:43.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.242653 28614 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/wal/instance:
21:03:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db9a128d9d4046bf915e0e521512c1d6"
21:03:43.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:43 on dist-test-slave-q92b"
21:03:43.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.262972 28614 fs_manager.cc:696] Time spent creating directory manager: real 0.020s user 0.014s sys 0.001s
21:03:43.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.282686 28639 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:43.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.292215 28614 fs_manager.cc:730] Time spent opening block manager: real 0.024s user 0.005s sys 0.004s
21:03:43.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.292632 28614 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/wal
21:03:43.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "db9a128d9d4046bf915e0e521512c1d6"
21:03:43.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:43 on dist-test-slave-q92b"
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.293072 28614 fs_report.cc:389] FS layout report
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/wal
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/wal
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/data/data
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:43.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:43.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.392549 28614 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.400012 28614 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:43.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.400571 28614 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:43.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.410338 28614 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:03:43.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.421952 28614 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:43.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.422243 28614 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:03:43.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.422526 28614 ts_tablet_manager.cc:662] Registered 0 tablets
21:03:43.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.422748 28614 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:03:43.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.662640 28614 rpc_server.cc:307] RPC server started. Bound to: 127.27.156.193:44843
21:03:43.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.662755 28751 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.156.193:44843 every 8 connection(s)
21:03:43.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.666985 28614 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/data/info.pb
21:03:43.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.677587 28275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 28614
21:03:43.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.678073 28275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-0/wal/instance
21:03:43.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:43 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690223, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.193@KRBTEST.COM for kudu/127.27.156.253@KRBTEST.COM
21:03:43.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:43 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690223, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.193@KRBTEST.COM for kudu/127.27.156.252@KRBTEST.COM
21:03:43.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.745482 28753 heartbeater.cc:345] Connected to a master server at 127.27.156.253:44373
21:03:43.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.745998 28753 heartbeater.cc:462] Registering TS with master...
21:03:43.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.156.194@KRBTEST.COM; defaulting to no policy
21:03:43.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.747771 28753 heartbeater.cc:508] Master 127.27.156.253:44373 requested a full tablet report, sending...
21:03:43.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:43.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.156.194@KRBTEST.COM" created.
21:03:43.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.755681 28423 ts_manager.cc:210] Registered new tserver with Master: db9a128d9d4046bf915e0e521512c1d6 (127.27.156.193:44843)
21:03:43.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.773308 28752 heartbeater.cc:345] Connected to a master server at 127.27.156.252:44355
21:03:43.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.773674 28752 heartbeater.cc:462] Registering TS with master...
21:03:43.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.774533 28752 heartbeater.cc:508] Master 127.27.156.252:44355 requested a full tablet report, sending...
21:03:43.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.777469 28509 ts_manager.cc:210] Registered new tserver with Master: db9a128d9d4046bf915e0e521512c1d6 (127.27.156.193:44843)
21:03:43.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.780566 28509 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.27.156.193@KRBTEST.COM'} at 127.27.156.193:47989
21:03:43.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:43 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690223, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.193@KRBTEST.COM for kudu/127.27.156.254@KRBTEST.COM
21:03:43.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.806777 28758 heartbeater.cc:345] Connected to a master server at 127.27.156.254:34367
21:03:43.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.807590 28758 heartbeater.cc:462] Registering TS with master...
21:03:43.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.808305 28758 heartbeater.cc:508] Master 127.27.156.254:34367 requested a full tablet report, sending...
21:03:43.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.812379 28340 ts_manager.cc:210] Registered new tserver with Master: db9a128d9d4046bf915e0e521512c1d6 (127.27.156.193:44843)
21:03:43.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:43.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:43.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:43.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:43.822708 28275 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.27.156.194: real 0.124s user 0.001s sys 0.013s
21:03:43.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.156.194@KRBTEST.COM; defaulting to no policy
21:03:43.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:43.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.156.194@KRBTEST.COM" created.
21:03:43.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:43.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:43.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:43.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:43.877585 28275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:43.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/wal
21:03:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/data
21:03:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/logs
21:03:43.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/data/info.pb
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.194:0
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.156.194
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.194
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.194
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:43.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:03:43.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:03:43.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:03:43.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:43.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:43.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:43.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:43.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:03:44.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:44 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690224, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:44.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:44.060501 28319 init.cc:295] Successfully reacquired a new kerberos TGT
21:03:44.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:44.376044 28766 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:44.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:44.376775 28766 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:44.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:44.377846 28766 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:44.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:44.450369 28766 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:44.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:44.451354 28766 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.156.194
21:03:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:44.502216 28766 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/logs
21:03:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:44.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/data
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/wal
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.194
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.194:0
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/data/info.pb
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.194
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.156.194
21:03:44.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:44.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:44.503867 28766 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:44.504832 28766 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:44.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:44.518153 28773 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:44.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:44.785218 28752 heartbeater.cc:500] Master 127.27.156.252:44355 was elected leader, sending a full tablet report...
21:03:46.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:44.519325 28774 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:46.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:45.922562 28772 debug-util.cc:398] Leaking SignalData structure 0x7b080002efa0 after lost signal to thread 28766
21:03:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:46.327064 28766 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.807s user 0.649s sys 1.045s
21:03:46.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:46.328699 28766 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.809s user 0.649s sys 1.045s
21:03:46.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:46.330384 28776 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:46.339896 28775 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1809 milliseconds
21:03:46.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.340026 28766 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:03:46.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.341605 28766 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:46.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.343792 28766 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:46.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.345175 28766 hybrid_clock.cc:648] HybridClock initialized: now 1794690226345124 us; error 43 us; skew 500 ppm
21:03:46.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.345986 28766 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:46.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:46 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690226, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:46.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.352035 28766 init.cc:377] Logged in from keytab as kudu/127.27.156.194@KRBTEST.COM (short username kudu)
21:03:46.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.357070 28766 webserver.cc:577] Webserver started at http://127.27.156.194:35413/ using document root <none> and password file <none>
21:03:46.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.358376 28766 fs_manager.cc:362] Metadata directory not provided
21:03:46.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.358649 28766 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:46.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.359192 28766 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:46.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.365337 28766 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/data/instance:
21:03:46.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f47ddb4d17a84ac5bdf0198ce72302be"
21:03:46.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:46 on dist-test-slave-q92b"
21:03:46.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.366859 28766 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/wal/instance:
21:03:46.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f47ddb4d17a84ac5bdf0198ce72302be"
21:03:46.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:46 on dist-test-slave-q92b"
21:03:46.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.376561 28766 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.005s
21:03:46.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.385084 28786 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:46.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.386504 28766 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.007s sys 0.000s
21:03:46.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.386898 28766 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/wal
21:03:46.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f47ddb4d17a84ac5bdf0198ce72302be"
21:03:46.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:46 on dist-test-slave-q92b"
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.387313 28766 fs_report.cc:389] FS layout report
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/wal
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/wal
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/data/data
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:46.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:46.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:46.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.459625 28766 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:46.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.473543 28766 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:46.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.474085 28766 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:46.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.477438 28766 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:03:46.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.482595 28766 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:46.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.482824 28766 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:03:46.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.483142 28766 ts_tablet_manager.cc:662] Registered 0 tablets
21:03:46.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.483352 28766 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:03:46.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:46 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690226, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:46.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.619443 28402 init.cc:295] Successfully reacquired a new kerberos TGT
21:03:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.667392 28766 rpc_server.cc:307] RPC server started. Bound to: 127.27.156.194:39397
21:03:46.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.667491 28899 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.156.194:39397 every 8 connection(s)
21:03:46.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.670365 28766 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/data/info.pb
21:03:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.678102 28275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 28766
21:03:46.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.678592 28275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-1/wal/instance
21:03:46.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.27.156.195@KRBTEST.COM; defaulting to no policy
21:03:46.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:46.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.27.156.195@KRBTEST.COM" created.
21:03:46.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:46 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690226, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.194@KRBTEST.COM for kudu/127.27.156.253@KRBTEST.COM
21:03:46.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:46 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690226, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.194@KRBTEST.COM for kudu/127.27.156.252@KRBTEST.COM
21:03:46.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:46 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690226, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.194@KRBTEST.COM for kudu/127.27.156.254@KRBTEST.COM
21:03:46.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.753389 28900 heartbeater.cc:345] Connected to a master server at 127.27.156.252:44355
21:03:46.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.753921 28900 heartbeater.cc:462] Registering TS with master...
21:03:46.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.755371 28900 heartbeater.cc:508] Master 127.27.156.252:44355 requested a full tablet report, sending...
21:03:46.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.756558 28901 heartbeater.cc:345] Connected to a master server at 127.27.156.253:44373
21:03:46.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.757030 28901 heartbeater.cc:462] Registering TS with master...
21:03:46.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.757279 28902 heartbeater.cc:345] Connected to a master server at 127.27.156.254:34367
21:03:46.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.757622 28902 heartbeater.cc:462] Registering TS with master...
21:03:46.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.758086 28901 heartbeater.cc:508] Master 127.27.156.253:44373 requested a full tablet report, sending...
21:03:46.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.758421 28902 heartbeater.cc:508] Master 127.27.156.254:34367 requested a full tablet report, sending...
21:03:46.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.758960 28509 ts_manager.cc:210] Registered new tserver with Master: f47ddb4d17a84ac5bdf0198ce72302be (127.27.156.194:39397)
21:03:46.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.760897 28509 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.27.156.194@KRBTEST.COM'} at 127.27.156.194:59891
21:03:46.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.761610 28340 ts_manager.cc:210] Registered new tserver with Master: f47ddb4d17a84ac5bdf0198ce72302be (127.27.156.194:39397)
21:03:46.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.762924 28423 ts_manager.cc:210] Registered new tserver with Master: f47ddb4d17a84ac5bdf0198ce72302be (127.27.156.194:39397)
21:03:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.27.156.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:46.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.27.156.195@KRBTEST.COM; defaulting to no policy
21:03:46.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:46.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.27.156.195@KRBTEST.COM" created.
21:03:46.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:03:46.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:46.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.27.156.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab.
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:46.807753 28275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/wal
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/data
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/logs
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/data/info.pb
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.195:0
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.156.195
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.195
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:46.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.195
21:03:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:03:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:03:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:03:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:46.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:03:47.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:47.147512 28914 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:47.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:47.148005 28914 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:47.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:47.148564 28914 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:47.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:47.183247 28914 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:47.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:47.184307 28914 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.156.195
21:03:47.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:47.223984 28914 tablet_server_runner.cc:78] Tablet server non-default flags:
21:03:47.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/logs
21:03:47.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:47.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:47.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:47.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:47.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/data
21:03:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/wal
21:03:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.195
21:03:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.195:0
21:03:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:47.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/data/info.pb
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.195
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.27.156.195
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:47.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:47.225705 28914 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:47.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:47.227109 28914 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:47.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:47.243443 28922 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:47.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:47.765242 28900 heartbeater.cc:500] Master 127.27.156.252:44355 was elected leader, sending a full tablet report...
21:03:48.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:47.248670 28921 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:48.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:47.251869 28925 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:48.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:48.743337 28923 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1495 milliseconds
21:03:48.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.743350 28914 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:03:48.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:48 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690228, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:48.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.747052 28488 init.cc:295] Successfully reacquired a new kerberos TGT
21:03:48.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.748487 28914 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:48.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.751443 28914 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:48.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.752880 28914 hybrid_clock.cc:648] HybridClock initialized: now 1794690228752811 us; error 80 us; skew 500 ppm
21:03:48.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.753705 28914 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:48.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:48 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690228, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:48.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.759924 28914 init.cc:377] Logged in from keytab as kudu/127.27.156.195@KRBTEST.COM (short username kudu)
21:03:48.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.765110 28914 webserver.cc:577] Webserver started at http://127.27.156.195:35721/ using document root <none> and password file <none>
21:03:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.766070 28914 fs_manager.cc:362] Metadata directory not provided
21:03:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.766263 28914 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:48.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.766637 28914 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
21:03:48.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.771322 28914 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/data/instance:
21:03:48.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "38023e392ffd42dbb1ae44d6ece9a281"
21:03:48.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:48 on dist-test-slave-q92b"
21:03:48.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.773633 28914 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/wal/instance:
21:03:48.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "38023e392ffd42dbb1ae44d6ece9a281"
21:03:48.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:48 on dist-test-slave-q92b"
21:03:48.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.781642 28914 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
21:03:48.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.788188 28936 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:48.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.790012 28914 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.004s
21:03:48.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.790298 28914 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/wal
21:03:48.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "38023e392ffd42dbb1ae44d6ece9a281"
21:03:48.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:48 on dist-test-slave-q92b"
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.790581 28914 fs_report.cc:389] FS layout report
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/wal
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/wal
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/data/data
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:48.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:48.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.846241 28914 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.851763 28914 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:48.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.852152 28914 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:48.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.855018 28914 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:03:48.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.859740 28914 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:03:48.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.859946 28914 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:03:48.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.860148 28914 ts_tablet_manager.cc:662] Registered 0 tablets
21:03:48.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:48.860283 28914 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:03:49.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.026786 28914 rpc_server.cc:307] RPC server started. Bound to: 127.27.156.195:39475
21:03:49.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.026845 29048 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.156.195:39475 every 8 connection(s)
21:03:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.029538 28914 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/data/info.pb
21:03:49.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.035820 28275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 28914
21:03:49.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.036348 28275 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/ts-2/wal/instance
21:03:49.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:49 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690228, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.195@KRBTEST.COM for kudu/127.27.156.253@KRBTEST.COM
21:03:49.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:49 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690228, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.195@KRBTEST.COM for kudu/127.27.156.252@KRBTEST.COM
21:03:49.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:49 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690228, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.195@KRBTEST.COM for kudu/127.27.156.254@KRBTEST.COM
21:03:49.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.134025 29049 heartbeater.cc:345] Connected to a master server at 127.27.156.252:44355
21:03:49.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.134471 29049 heartbeater.cc:462] Registering TS with master...
21:03:49.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.134497 29050 heartbeater.cc:345] Connected to a master server at 127.27.156.253:44373
21:03:49.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.135022 29050 heartbeater.cc:462] Registering TS with master...
21:03:49.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.135908 29049 heartbeater.cc:508] Master 127.27.156.252:44355 requested a full tablet report, sending...
21:03:49.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.136013 29050 heartbeater.cc:508] Master 127.27.156.253:44373 requested a full tablet report, sending...
21:03:49.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.139961 28509 ts_manager.cc:210] Registered new tserver with Master: 38023e392ffd42dbb1ae44d6ece9a281 (127.27.156.195:39475)
21:03:49.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.140110 28423 ts_manager.cc:210] Registered new tserver with Master: 38023e392ffd42dbb1ae44d6ece9a281 (127.27.156.195:39475)
21:03:49.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.142776 28509 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.27.156.195@KRBTEST.COM'} at 127.27.156.195:34309
21:03:49.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.151505 29051 heartbeater.cc:345] Connected to a master server at 127.27.156.254:34367
21:03:49.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.151865 29051 heartbeater.cc:462] Registering TS with master...
21:03:49.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.152969 29051 heartbeater.cc:508] Master 127.27.156.254:34367 requested a full tablet report, sending...
21:03:49.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.155454 28340 ts_manager.cc:210] Registered new tserver with Master: 38023e392ffd42dbb1ae44d6ece9a281 (127.27.156.195:39475)
21:03:49.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.159348 28275 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
21:03:49.176 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:49.177 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
21:03:49.185 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
21:03:49.186 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
21:03:49.188 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.27.156.254"
port: 34367
}
bound_http_address {
host: "127.27.156.254"
port: 39973
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.27.156.253"
port: 44373
}
bound_http_address {
host: "127.27.156.253"
port: 39629
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.27.156.252"
port: 44355
}
bound_http_address {
host: "127.27.156.252"
port: 44327
}
}
}
21:03:49.188 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:03:49.190 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.27.156.193"
port: 44843
}
bound_http_address {
host: "127.27.156.193"
port: 33359
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.27.156.194"
port: 39397
}
bound_http_address {
host: "127.27.156.194"
port: 35413
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.27.156.195"
port: 39475
}
bound_http_address {
host: "127.27.156.195"
port: 35721
}
}
}
21:03:49.190 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:03:49.191 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:03:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:49 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690229, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:49.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:03:49.229 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:49.231 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc
21:03:49.239 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:03:49.240 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc
21:03:49.241 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:03:49.266 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
21:03:49.268 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:03:49.268 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:03:49.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:49 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690229, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:49.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:03:49.313 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:49.314 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc
21:03:49.315 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:03:49.315 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.156.254:34367
21:03:49.316 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:03:49.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.317458 28275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 28304
21:03:49.350 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:49.350 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.156.252:44355
21:03:49.350 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:03:49.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.350854 28275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 28470
21:03:49.383 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:49.383 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.27.156.253:44373
21:03:49.384 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:03:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.385385 28275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 28385
21:03:49.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:49.417 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.156.254:34367
21:03:49.422 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:03:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.424098 28275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal
21:03:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data
21:03:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/logs
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data/info.pb
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.254:34367
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.254
21:03:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39973
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.254
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:03:49.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:03:49.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.773003 29064 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:49.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.773773 29064 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:49.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.775003 29064 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:49.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.778584 28902 heartbeater.cc:648] Failed to heartbeat to 127.27.156.254:34367 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.27.156.254:34367: connect: Connection refused (error 111)
21:03:49.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.794401 28753 heartbeater.cc:648] Failed to heartbeat to 127.27.156.253:44373 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:49.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.814308 29064 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
21:03:49.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.814627 29064 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:03:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.815083 29064 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:49.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.815469 29064 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.815819 29064 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:03:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.816151 29064 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.816444 29064 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37809/jwks.json
21:03:49.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.858664 29064 master_runner.cc:387] Master server non-default flags:
21:03:49.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/logs
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.254
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.254:34367
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:49.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data/info.pb
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.254
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39973
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:49.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.860371 29064 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:49.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:49.861842 29064 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:49.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.872579 29073 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:50.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:50.148149 29050 heartbeater.cc:648] Failed to heartbeat to 127.27.156.253:44373 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:50.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:50.151988 29049 heartbeater.cc:500] Master 127.27.156.252:44355 was elected leader, sending a full tablet report...
21:03:51.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:49.874261 29074 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:51.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.233948 29076 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:51.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.236460 29075 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1358 milliseconds
21:03:51.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.236590 29064 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:03:51.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.237967 29064 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:51.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.240374 29064 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:51.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.241714 29064 hybrid_clock.cc:648] HybridClock initialized: now 1794690231241677 us; error 48 us; skew 500 ppm
21:03:51.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.242568 29064 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:51.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:51 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690231, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:51.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.248720 29064 init.cc:377] Logged in from keytab as kudu/127.27.156.254@KRBTEST.COM (short username kudu)
21:03:51.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.254046 29064 webserver.cc:577] Webserver started at http://127.27.156.254:39973/ using document root <none> and password file <none>
21:03:51.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.255286 29064 fs_manager.cc:362] Metadata directory not provided
21:03:51.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.255569 29064 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.266278 29064 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.007s sys 0.001s
21:03:51.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.272027 29092 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:51.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.273497 29064 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
21:03:51.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.273881 29064 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal
21:03:51.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ef2ace3461a84b6a82ef8b5c267c7bff"
21:03:51.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:34 on dist-test-slave-q92b"
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.276643 29064 fs_report.cc:389] FS layout report
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/wal
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data/data
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:51.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:51.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:51.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:51.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.343118 29064 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.349083 29064 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:51.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.349566 29064 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.430075 29064 rpc_server.cc:307] RPC server started. Bound to: 127.27.156.254:34367
21:03:51.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.430142 29143 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.156.254:34367 every 8 connection(s)
21:03:51.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.432992 29064 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-0/data/info.pb
21:03:51.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.439443 28275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 29064
21:03:51.440 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:51.440 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.156.252:44355
21:03:51.440 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.441991 28275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/logs
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data/info.pb
21:03:51.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:51.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:51.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:51.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:51.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:51.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:51.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:51.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:51.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:51.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.252:44355
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.252
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44327
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.252
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:51.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:03:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:03:51.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.444880 29144 sys_catalog.cc:263] Verifying existing consensus state
21:03:51.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.446924 28119 debug-util.cc:398] Leaking SignalData structure 0x7b08000bae60 after lost signal to thread 27994
21:03:51.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.447999 28119 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad7a0 after lost signal to thread 28122
21:03:51.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.452491 29144 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: Bootstrap starting.
21:03:51.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.511962 29144 log.cc:826] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: Log is configured to *not* fsync() on all Append() calls
21:03:51.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.550904 29144 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:03:51.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.552529 29144 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: Bootstrap complete.
21:03:51.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.592197 29144 raft_consensus.cc:360] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:51.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.596232 29144 raft_consensus.cc:748] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: ef2ace3461a84b6a82ef8b5c267c7bff, State: Initialized, Role: FOLLOWER
21:03:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.597482 29144 consensus_queue.cc:261] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:51.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.600464 29148 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:51.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.601454 29148 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: This master's current role is: FOLLOWER
21:03:51.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.602599 29144 sys_catalog.cc:565] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: configured and running, proceeding with master startup.
21:03:51.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.649109 29159 catalog_manager.cc:1310] Loaded cluster ID: 7c1de3e32ddb4085aac5187ca386de05
21:03:51.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.650839 29159 catalog_manager.cc:1603] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: loading cluster ID for follower catalog manager: success
21:03:51.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.659035 29159 catalog_manager.cc:1625] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: acquiring CA information for follower catalog manager: success
21:03:51.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.678256 29159 catalog_manager.cc:1653] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:03:51.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:51 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690231, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.254@KRBTEST.COM for kudu/127.27.156.254@KRBTEST.COM
21:03:51.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.707365 29064 master_runner.cc:187] Error getting master registration for 127.27.156.253:44373: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111),
21:03:51.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.735065 29064 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:03:51.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.815184 28902 heartbeater.cc:345] Connected to a master server at 127.27.156.254:34367
21:03:51.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.821799 29109 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "f47ddb4d17a84ac5bdf0198ce72302be" instance_seqno: 1794690226611728) as {username='kudu', principal='kudu/127.27.156.194@KRBTEST.COM'} at 127.27.156.194:48305; Asking this server to re-register.
21:03:51.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.823706 28902 heartbeater.cc:462] Registering TS with master...
21:03:51.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.827451 29109 ts_manager.cc:210] Registered new tserver with Master: f47ddb4d17a84ac5bdf0198ce72302be (127.27.156.194:39397)
21:03:51.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.855631 28758 heartbeater.cc:345] Connected to a master server at 127.27.156.254:34367
21:03:51.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.858726 29109 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "db9a128d9d4046bf915e0e521512c1d6" instance_seqno: 1794690223603738) as {username='kudu', principal='kudu/127.27.156.193@KRBTEST.COM'} at 127.27.156.193:58205; Asking this server to re-register.
21:03:51.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.860195 28758 heartbeater.cc:462] Registering TS with master...
21:03:51.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.863109 29109 ts_manager.cc:210] Registered new tserver with Master: db9a128d9d4046bf915e0e521512c1d6 (127.27.156.193:44843)
21:03:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.875278 29146 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.875952 29146 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:51.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.876948 29146 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:51.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.912470 29146 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
21:03:51.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.912767 29146 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:03:51.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.913251 29146 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:51.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.913633 29146 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:51.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.914023 29146 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:03:51.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.914367 29146 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:51.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.914666 29146 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37809/jwks.json
21:03:51.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.957461 29146 master_runner.cc:387] Master server non-default flags:
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/logs
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.252
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.252:44355
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:51.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:03:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:03:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data/info.pb
21:03:51.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.252
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44327
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.959234 29146 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:51.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:51.960625 29146 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.973318 29172 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:52.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:52.188440 29051 heartbeater.cc:345] Connected to a master server at 127.27.156.254:34367
21:03:52.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:52.191309 29109 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "38023e392ffd42dbb1ae44d6ece9a281" instance_seqno: 1794690228988620) as {username='kudu', principal='kudu/127.27.156.195@KRBTEST.COM'} at 127.27.156.195:55315; Asking this server to re-register.
21:03:52.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:52.192946 29051 heartbeater.cc:462] Registering TS with master...
21:03:52.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:52.196444 29109 ts_manager.cc:210] Registered new tserver with Master: 38023e392ffd42dbb1ae44d6ece9a281 (127.27.156.195:39475)
21:03:52.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:52.760833 29179 raft_consensus.cc:499] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:52.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:52.761745 29179 raft_consensus.cc:521] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:52.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:52.774185 29179 leader_election.cc:290] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b1a43974a60e4addb0042d681f0649e4 (127.27.156.253:44373), 78a99a0f6f524309b49ee7391870993d (127.27.156.252:44355)
21:03:52.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:52.776937 29094 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:52.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:52.788666 29093 leader_election.cc:336] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b1a43974a60e4addb0042d681f0649e4 (127.27.156.253:44373): Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:52.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:52.789175 29094 leader_election.cc:336] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 78a99a0f6f524309b49ee7391870993d (127.27.156.252:44355): Network error: Client connection negotiation failed: client connection to 127.27.156.252:44355: connect: Connection refused (error 111)
21:03:52.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:52.789827 29094 leader_election.cc:304] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ef2ace3461a84b6a82ef8b5c267c7bff; no voters: 78a99a0f6f524309b49ee7391870993d, b1a43974a60e4addb0042d681f0649e4
21:03:52.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:52.791028 29179 raft_consensus.cc:2771] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:03:53.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:53.081067 28896 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae0c0 after lost signal to thread 28767
21:03:53.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:53.082267 28896 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6440 after lost signal to thread 28899
21:03:53.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:53 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690233, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:53.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.232240 28635 init.cc:295] Successfully reacquired a new kerberos TGT
21:03:53.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:53.376984 29171 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 29146
21:03:53.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:53.818565 29146 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 1.847s user 0.549s sys 0.920s
21:03:53.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:53.818931 29146 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 1.847s user 0.549s sys 0.920s
21:03:53.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:51.974123 29173 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:53.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:53.831110 29191 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:53.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.834178 29146 server_base.cc:1061] running on GCE node
21:03:53.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.836139 29146 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:53.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.855192 29146 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:53.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.857208 29146 hybrid_clock.cc:648] HybridClock initialized: now 1794690233857148 us; error 36 us; skew 500 ppm
21:03:53.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.858155 29146 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:53.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:53 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690233, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:53.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.874297 29146 init.cc:377] Logged in from keytab as kudu/127.27.156.252@KRBTEST.COM (short username kudu)
21:03:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.882025 29146 webserver.cc:577] Webserver started at http://127.27.156.252:44327/ using document root <none> and password file <none>
21:03:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.883361 29146 fs_manager.cc:362] Metadata directory not provided
21:03:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.883661 29146 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:53.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.897125 29146 fs_manager.cc:714] Time spent opening directory manager: real 0.008s user 0.005s sys 0.001s
21:03:53.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.903609 29201 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:53.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.906498 29146 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.003s sys 0.003s
21:03:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.907016 29146 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal
21:03:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "78a99a0f6f524309b49ee7391870993d"
21:03:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:38 on dist-test-slave-q92b"
21:03:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.909973 29146 fs_report.cc:389] FS layout report
21:03:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal
21:03:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/wal
21:03:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data/data
21:03:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:53.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:53.995198 29146 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:54.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.003252 29146 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:54.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.003832 29146 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:54.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.121845 29252 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.156.252:44355 every 8 connection(s)
21:03:54.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.121863 29146 rpc_server.cc:307] RPC server started. Bound to: 127.27.156.252:44355
21:03:54.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.126022 29146 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-2/data/info.pb
21:03:54.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.135731 28275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 29146
21:03:54.137 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:54.138 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.27.156.253:44373
21:03:54.138 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:03:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.141589 28275 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu
21:03:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal
21:03:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data
21:03:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:03:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:03:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:03:54.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:54.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/logs
21:03:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data/info.pb
21:03:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:03:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:03:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:03:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:03:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:54.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.253:44373
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.253
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39629
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.253
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:03:54.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:03:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:54.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:54.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:54.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:54.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:54.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:54.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:54.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:54.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:03:54.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:03:54.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.173444 29253 sys_catalog.cc:263] Verifying existing consensus state
21:03:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.185671 29253 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: Bootstrap starting.
21:03:54.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.274715 29253 log.cc:826] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: Log is configured to *not* fsync() on all Append() calls
21:03:54.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.300921 29049 heartbeater.cc:345] Connected to a master server at 127.27.156.252:44355
21:03:54.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.301448 29049 heartbeater.cc:500] Master 127.27.156.252:44355 was elected leader, sending a full tablet report...
21:03:54.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:54.309718 27979 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad220 after lost signal to thread 27853
21:03:54.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:54.310858 27979 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad9e0 after lost signal to thread 27982
21:03:54.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.402130 29253 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:03:54.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.410092 29253 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: Bootstrap complete.
21:03:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.491930 29253 raft_consensus.cc:360] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:54.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.505599 29253 raft_consensus.cc:748] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 78a99a0f6f524309b49ee7391870993d, State: Initialized, Role: FOLLOWER
21:03:54.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.506729 29253 consensus_queue.cc:261] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.523072 29261 sys_catalog.cc:455] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:54.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.524051 29261 sys_catalog.cc:458] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: This master's current role is: FOLLOWER
21:03:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.528676 29253 sys_catalog.cc:565] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: configured and running, proceeding with master startup.
21:03:54.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.621259 29273 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:03:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:54 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690233, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.252@KRBTEST.COM for kudu/127.27.156.254@KRBTEST.COM
21:03:54.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.721086 29146 master_runner.cc:187] Error getting master registration for 127.27.156.253:44373: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111),
21:03:54.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.752566 29146 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:03:54.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:54 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690233, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.193@KRBTEST.COM for kudu/127.27.156.252@KRBTEST.COM
21:03:54.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.909633 28900 heartbeater.cc:345] Connected to a master server at 127.27.156.252:44355
21:03:54.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.916637 29218 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "f47ddb4d17a84ac5bdf0198ce72302be" instance_seqno: 1794690226611728) as {username='kudu', principal='kudu/127.27.156.194@KRBTEST.COM'} at 127.27.156.194:35023; Asking this server to re-register.
21:03:54.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.918533 28900 heartbeater.cc:462] Registering TS with master...
21:03:54.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.922935 29217 ts_manager.cc:210] Registered new tserver with Master: f47ddb4d17a84ac5bdf0198ce72302be (127.27.156.194:39397)
21:03:54.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.928663 28752 heartbeater.cc:345] Connected to a master server at 127.27.156.252:44355
21:03:54.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.931437 29217 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "db9a128d9d4046bf915e0e521512c1d6" instance_seqno: 1794690223603738) as {username='kudu', principal='kudu/127.27.156.193@KRBTEST.COM'} at 127.27.156.193:59633; Asking this server to re-register.
21:03:54.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.932925 28752 heartbeater.cc:462] Registering TS with master...
21:03:54.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:54.935863 29217 ts_manager.cc:210] Registered new tserver with Master: db9a128d9d4046bf915e0e521512c1d6 (127.27.156.193:44843)
21:03:54.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:54.967724 29255 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
21:03:54.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:54.968400 29255 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:03:54.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:54.969175 29255 flags.cc:432] Enabled unsafe flag: --never_fsync=true
21:03:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.006222 29255 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
21:03:55.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.006618 29255 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
21:03:55.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.007237 29255 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
21:03:55.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.007732 29255 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
21:03:55.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.008209 29255 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
21:03:55.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.008693 29255 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
21:03:55.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.009024 29255 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:37809/jwks.json
21:03:55.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.037250 29287 raft_consensus.cc:499] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:03:55.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.042279 29287 raft_consensus.cc:521] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:55.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.063278 29287 leader_election.cc:290] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers b1a43974a60e4addb0042d681f0649e4 (127.27.156.253:44373), 78a99a0f6f524309b49ee7391870993d (127.27.156.252:44355)
21:03:55.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.069362 29093 leader_election.cc:336] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b1a43974a60e4addb0042d681f0649e4 (127.27.156.253:44373): Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:55.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:55 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690231, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.254@KRBTEST.COM for kudu/127.27.156.252@KRBTEST.COM
21:03:55.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.092916 29228 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "78a99a0f6f524309b49ee7391870993d" is_pre_election: true
21:03:55.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.093799 29228 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ef2ace3461a84b6a82ef8b5c267c7bff in term 1.
21:03:55.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.094480 29255 master_runner.cc:387] Master server non-default flags:
21:03:55.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/logs
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.27.156.212:43197
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.27.156.254:34367,127.27.156.253:44373,127.27.156.252:44355
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/krb5kdc/kudu.keytab
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.27.156.253
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.27.156.253:44373
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37809/jwks.json
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data/info.pb
21:03:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.27.156.253
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=39629
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3458f03c6ee765ff81fe1ff179b7d6d82b82420b
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Nov 2026 20:43:23 UTC on e7f111948823
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14786
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.095345 29094 leader_election.cc:304] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 78a99a0f6f524309b49ee7391870993d, ef2ace3461a84b6a82ef8b5c267c7bff; no voters: b1a43974a60e4addb0042d681f0649e4
21:03:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.096485 29287 raft_consensus.cc:2826] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 1 FOLLOWER]: Leader pre-election won for term 2
21:03:55.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.096814 29287 raft_consensus.cc:499] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:03:55.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.097124 29287 raft_consensus.cc:3096] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 1 FOLLOWER]: Advancing to term 2
21:03:55.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.107276 29287 raft_consensus.cc:521] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:55.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.096263 29255 env_posix.cc:2314] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:03:55.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.109759 29255 file_cache.cc:504] Constructed file cache file cache with capacity 419430
21:03:55.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.114223 29093 leader_election.cc:336] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer b1a43974a60e4addb0042d681f0649e4 (127.27.156.253:44373): Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111)
21:03:55.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.116513 29228 tablet_service.cc:1906] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "78a99a0f6f524309b49ee7391870993d"
21:03:55.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.117072 29228 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 1 FOLLOWER]: Advancing to term 2
21:03:55.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.137315 29287 leader_election.cc:290] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [CANDIDATE]: Term 2 election: Requested vote from peers b1a43974a60e4addb0042d681f0649e4 (127.27.156.253:44373), 78a99a0f6f524309b49ee7391870993d (127.27.156.252:44355)
21:03:55.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.139209 29228 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ef2ace3461a84b6a82ef8b5c267c7bff in term 2.
21:03:55.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.140568 29094 leader_election.cc:304] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 78a99a0f6f524309b49ee7391870993d, ef2ace3461a84b6a82ef8b5c267c7bff; no voters: b1a43974a60e4addb0042d681f0649e4
21:03:55.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.141975 29287 raft_consensus.cc:2826] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 2 FOLLOWER]: Leader election won for term 2
21:03:55.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.147469 29292 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:55.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.150398 29287 raft_consensus.cc:705] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [term 2 LEADER]: Becoming Leader. State: Replica: ef2ace3461a84b6a82ef8b5c267c7bff, State: Running, Role: LEADER
21:03:55.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.152036 29287 consensus_queue.cc:238] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:55.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.178036 29293 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: SysCatalogTable state changed. Reason: New leader ef2ace3461a84b6a82ef8b5c267c7bff. Latest consensus state: current_term: 2 leader_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:55.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.180342 29293 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: This master's current role is: LEADER
21:03:55.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.187829 29298 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
21:03:55.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.191502 29228 raft_consensus.cc:1285] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [term 2 FOLLOWER]: Refusing update from remote peer ef2ace3461a84b6a82ef8b5c267c7bff: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
21:03:55.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.194450 29297 consensus_queue.cc:1049] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [LEADER]: Connected to new peer: Peer: permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
21:03:55.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.211073 29093 consensus_peers.cc:597] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff -> Peer b1a43974a60e4addb0042d681f0649e4 (127.27.156.253:44373): Couldn't send request to peer b1a43974a60e4addb0042d681f0649e4. Status: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:03:55.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.239365 29298 catalog_manager.cc:1535] Initializing Kudu cluster ID...
21:03:55.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.270154 29297 sys_catalog.cc:455] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:55.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.275127 29297 sys_catalog.cc:458] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff [sys.catalog]: This master's current role is: LEADER
21:03:55.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.280467 29298 catalog_manager.cc:1310] Loaded cluster ID: 7c1de3e32ddb4085aac5187ca386de05
21:03:55.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.286881 29302 sys_catalog.cc:455] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: SysCatalogTable state changed. Reason: New leader ef2ace3461a84b6a82ef8b5c267c7bff. Latest consensus state: current_term: 2 leader_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:55.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.297341 29302 sys_catalog.cc:458] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: This master's current role is: FOLLOWER
21:03:55.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.289618 29306 sys_catalog.cc:455] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:55.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.298851 29306 sys_catalog.cc:458] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d [sys.catalog]: This master's current role is: FOLLOWER
21:03:55.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.299639 29298 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
21:03:55.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.305006 29049 heartbeater.cc:500] Master 127.27.156.252:44355 was elected leader, sending a full tablet report...
21:03:55.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.307881 29217 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "38023e392ffd42dbb1ae44d6ece9a281" instance_seqno: 1794690228988620) as {username='kudu', principal='kudu/127.27.156.195@KRBTEST.COM'} at 127.27.156.195:59841; Asking this server to re-register.
21:03:55.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.309686 29049 heartbeater.cc:462] Registering TS with master...
21:03:55.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.310195 29298 catalog_manager.cc:1555] Loading token signing keys...
21:03:55.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.313364 29217 ts_manager.cc:210] Registered new tserver with Master: 38023e392ffd42dbb1ae44d6ece9a281 (127.27.156.195:39475)
21:03:55.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.314783 29298 catalog_manager.cc:6232] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff: Loaded TSK: 0
21:03:55.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.316089 29298 catalog_manager.cc:1565] Initializing in-progress tserver states...
21:03:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.650059 29273 catalog_manager.cc:1310] Loaded cluster ID: 7c1de3e32ddb4085aac5187ca386de05
21:03:55.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.650370 29273 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: loading cluster ID for follower catalog manager: success
21:03:55.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.668812 29273 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: acquiring CA information for follower catalog manager: success
21:03:55.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:55.687990 29273 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 78a99a0f6f524309b49ee7391870993d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:03:56.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:56 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690236, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:56.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:56.357234 28782 init.cc:295] Successfully reacquired a new kerberos TGT
21:03:56.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:56.745788 29064 master_runner.cc:187] Error getting master registration for 127.27.156.253:44373: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111),
21:03:56.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:56.746274 29064 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:03:56.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:56.772964 29045 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae600 after lost signal to thread 28915
21:03:56.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:56.774542 29045 debug-util.cc:398] Leaking SignalData structure 0x7b08000b74a0 after lost signal to thread 29048
21:03:56.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:56.878827 28902 heartbeater.cc:500] Master 127.27.156.254:34367 was elected leader, sending a full tablet report...
21:03:56.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:56.885507 28758 heartbeater.cc:500] Master 127.27.156.254:34367 was elected leader, sending a full tablet report...
21:03:57.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:57.243876 29051 heartbeater.cc:500] Master 127.27.156.254:34367 was elected leader, sending a full tablet report...
21:03:57.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:57.768469 29093 consensus_peers.cc:597] T 00000000000000000000000000000000 P ef2ace3461a84b6a82ef8b5c267c7bff -> Peer b1a43974a60e4addb0042d681f0649e4 (127.27.156.253:44373): Couldn't send request to peer b1a43974a60e4addb0042d681f0649e4. Status: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:03:58.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:58.144299 29093 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.156.253:44373: connect: Connection refused (error 111) [suppressed 9 similar messages]
21:03:58.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:58 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690238, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:58.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.765995 28932 init.cc:295] Successfully reacquired a new kerberos TGT
21:03:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:56.544902 29291 debug-util.cc:398] Leaking SignalData structure 0x7b0800037780 after lost signal to thread 29255
21:03:58.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:55.152383 29294 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:58.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:58.805655 29255 thread.cc:642] GCE (cloud detector) Time spent creating pthread: real 3.655s user 0.936s sys 1.664s
21:03:58.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:58.806954 29255 thread.cc:609] GCE (cloud detector) Time spent starting thread: real 3.657s user 0.936s sys 1.664s
21:03:58.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.824362 29255 server_base.cc:1061] running on GCE node
21:03:58.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:58.826381 29328 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:03:58.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.831758 29255 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:03:58.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.839095 29255 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:03:58.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.840582 29255 hybrid_clock.cc:648] HybridClock initialized: now 1794690238840541 us; error 46 us; skew 500 ppm
21:03:58.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.842303 29255 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:03:58.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:58 dist-test-slave-q92b krb5kdc[28287](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690238, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:03:58.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.852697 29255 init.cc:377] Logged in from keytab as kudu/127.27.156.253@KRBTEST.COM (short username kudu)
21:03:58.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.899511 29255 webserver.cc:577] Webserver started at http://127.27.156.253:39629/ using document root <none> and password file <none>
21:03:58.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.901368 29255 fs_manager.cc:362] Metadata directory not provided
21:03:58.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.901880 29255 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:03:58.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.927660 29255 fs_manager.cc:714] Time spent opening directory manager: real 0.021s user 0.005s sys 0.013s
21:03:58.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.943115 29334 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:03:58.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.945091 29255 fs_manager.cc:730] Time spent opening block manager: real 0.014s user 0.001s sys 0.009s
21:03:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.945824 29255 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data,/tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal
21:03:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b1a43974a60e4addb0042d681f0649e4"
21:03:58.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-14 21:03:36 on dist-test-slave-q92b"
21:03:58.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:58.955019 29255 fs_report.cc:389] FS layout report
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/wal
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data/data
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:03:58.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:03:59.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.052206 29255 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:03:59.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.061436 29255 env_posix.cc:2314] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:03:59.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.062371 29255 kserver.cc:163] Server-wide thread pool size limit: 3276
21:03:59.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.228091 29255 rpc_server.cc:307] RPC server started. Bound to: 127.27.156.253:44373
21:03:59.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.228157 29388 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.27.156.253:44373 every 8 connection(s)
21:03:59.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.231293 29255 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskerlxO1/test-tmp/mini-kudu-cluster3768695652227171538/master-1/data/info.pb
21:03:59.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.240237 28275 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu as pid 29255
21:03:59.241 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:03:59.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.275835 29389 sys_catalog.cc:263] Verifying existing consensus state
21:03:59.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.288939 29389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: Bootstrap starting.
21:03:59.333 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.156.254:34367(127.27.156.254:34367)] connecting to peer
21:03:59.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.365881 29389 log.cc:826] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: Log is configured to *not* fsync() on all Append() calls
21:03:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.425684 29389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:03:59.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.431452 29389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: Bootstrap complete.
21:03:59.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:59 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690238, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.195@KRBTEST.COM for kudu/127.27.156.253@KRBTEST.COM
21:03:59.493 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.156.252:44355(127.27.156.252:44355)] connecting to peer
21:03:59.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.509723 29050 heartbeater.cc:345] Connected to a master server at 127.27.156.253:44373
21:03:59.516 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.27.156.253:44373(127.27.156.253:44373)] connecting to peer
21:03:59.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.522688 29389 raft_consensus.cc:360] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:59.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.541851 29389 raft_consensus.cc:748] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b1a43974a60e4addb0042d681f0649e4, State: Initialized, Role: FOLLOWER
21:03:59.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.543268 29389 consensus_queue.cc:261] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } }
21:03:59.551 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.27.156.252:44355(127.27.156.252:44355)] Successfully connected to peer
21:03:59.553 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.27.156.254:34367(127.27.156.254:34367)] Successfully connected to peer
21:03:59.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.555586 29398 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:59.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.556854 29398 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: This master's current role is: FOLLOWER
21:03:59.557 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.27.156.253:44373(127.27.156.253:44373)] Successfully connected to peer
21:03:59.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.564632 29389 sys_catalog.cc:565] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: configured and running, proceeding with master startup.
21:03:59.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:59 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690231, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.254@KRBTEST.COM for kudu/127.27.156.253@KRBTEST.COM
21:03:59.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.724999 29364 raft_consensus.cc:3096] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [term 1 FOLLOWER]: Advancing to term 2
21:03:59.773 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
21:03:59.779 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.27.156.252:44355(127.27.156.252:44355)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:03:59.780 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.27.156.252:44355: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:03:59.782 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.27.156.253:44373(127.27.156.253:44373)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:03:59.782 [INFO - kudu-nio-2] (ConnectToCluster.java:385) Unable to connect to master 127.27.156.253:44373: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:03:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:59.781673 29399 negotiation.cc:336] Failed RPC negotiation. Trace:
21:03:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.550115 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:33978 (local address 127.27.156.252:44355)
21:03:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.560703 (+ 10588us) server_negotiation.cc:207] Beginning negotiation
21:03:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.560740 (+ 37us) server_negotiation.cc:400] Waiting for connection header
21:03:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.668494 (+107754us) server_negotiation.cc:408] Connection header received
21:03:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.763351 (+ 94857us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:03:59.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.763397 (+ 46us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:03:59.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.763699 (+ 302us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:03:59.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.763930 (+ 231us) server_negotiation.cc:227] Negotiated authn=SASL
21:03:59.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.781478 (+ 17548us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:33978: BlockingRecv error: recv got EOF from 127.0.0.1:33978 (error 108)
21:03:59.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":9762,"thread_start_us":9486,"threads_started":1}
21:03:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:59.783607 29394 negotiation.cc:336] Failed RPC negotiation. Trace:
21:03:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.557840 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:34918 (local address 127.27.156.253:44373)
21:03:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.558802 (+ 962us) server_negotiation.cc:207] Beginning negotiation
21:03:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.558837 (+ 35us) server_negotiation.cc:400] Waiting for connection header
21:03:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.667893 (+109056us) server_negotiation.cc:408] Connection header received
21:03:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.737598 (+ 69705us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:03:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.737637 (+ 39us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:03:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.737889 (+ 252us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:03:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.738075 (+ 186us) server_negotiation.cc:227] Negotiated authn=SASL
21:03:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.783319 (+ 45244us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:34918: BlockingRecv error: recv got EOF from 127.0.0.1:34918 (error 108)
21:03:59.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":163}
21:03:59.787 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.27.156.254:34367(127.27.156.254:34367)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:03:59.788 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.27.156.254:34367: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:03:59.791 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.27.156.252:44355(127.27.156.252:44355)] handling channelInactive
21:03:59.791 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.27.156.253:44373(127.27.156.253:44373)] handling channelInactive
21:03:59.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:59 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690233, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.252@KRBTEST.COM for kudu/127.27.156.253@KRBTEST.COM
21:03:59.796 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=549), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.156.254:34367, [234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.156.252:44355, [257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.156.253:44373, [522ms] Sub RPC ConnectToMaster: received response from server master-127.27.156.252:44355: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [524ms] Sub RPC ConnectToMaster: received response from server master-127.27.156.253:44373: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [530ms] Sub RPC ConnectToMaster: received response from server master-127.27.156.254:34367: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.27.156.254:34367,127.27.156.252:44355,127.27.156.253:44373). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.27.156.254:34367,127.27.156.252:44355,127.27.156.253:44373). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:840)
21:03:59.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.816759 29146 master_runner.cc:187] Error getting master registration for 127.27.156.253:44373: OK, instance_id { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" instance_seqno: 1794690239197540 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:03:59.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.817601 29146 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
21:03:59.818 [DEBUG - kudu-nio-0] (KuduRpc.java:274) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=573), Traces: [0ms] refreshing cache from master, [49ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.156.254:34367, [234ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.156.252:44355, [257ms] Sub RPC ConnectToMaster: sending RPC to server master-127.27.156.253:44373, [522ms] Sub RPC ConnectToMaster: received response from server master-127.27.156.252:44355: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [524ms] Sub RPC ConnectToMaster: received response from server master-127.27.156.253:44373: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [530ms] Sub RPC ConnectToMaster: received response from server master-127.27.156.254:34367: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
21:03:59.822 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.156.254:34367(127.27.156.254:34367)] disconnecting while in state TERMINATED
21:03:59.824 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.156.252:44355(127.27.156.252:44355)] disconnecting while in state TERMINATED
21:03:59.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.824047 29398 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ef2ace3461a84b6a82ef8b5c267c7bff. Latest consensus state: current_term: 2 leader_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:59.825 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.27.156.253:44373(127.27.156.253:44373)] disconnecting while in state TERMINATED
21:03:59.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.824747 29398 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: This master's current role is: FOLLOWER
21:03:59.830 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:03:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.828365 29416 sys_catalog.cc:455] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ef2ace3461a84b6a82ef8b5c267c7bff" member_type: VOTER last_known_addr { host: "127.27.156.254" port: 34367 } } peers { permanent_uuid: "b1a43974a60e4addb0042d681f0649e4" member_type: VOTER last_known_addr { host: "127.27.156.253" port: 44373 } } peers { permanent_uuid: "78a99a0f6f524309b49ee7391870993d" member_type: VOTER last_known_addr { host: "127.27.156.252" port: 44355 } } }
21:03:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.829213 29416 sys_catalog.cc:458] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4 [sys.catalog]: This master's current role is: FOLLOWER
21:03:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261114 21:03:59.828675 29400 negotiation.cc:336] Failed RPC negotiation. Trace:
21:03:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.553318 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:41494 (local address 127.27.156.254:34367)
21:03:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.561918 (+ 8600us) server_negotiation.cc:207] Beginning negotiation
21:03:59.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.561948 (+ 30us) server_negotiation.cc:400] Waiting for connection header
21:03:59.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.668276 (+106328us) server_negotiation.cc:408] Connection header received
21:03:59.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.736021 (+ 67745us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:03:59.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.736054 (+ 33us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:03:59.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.736294 (+ 240us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:03:59.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.736527 (+ 233us) server_negotiation.cc:227] Negotiated authn=SASL
21:03:59.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1114 21:03:59.828493 (+ 91966us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41494: BlockingRecv error: recv got EOF from 127.0.0.1:41494 (error 108)
21:03:59.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":8060,"thread_start_us":7797,"threads_started":1}
21:03:59.842 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:03:59.842 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.27.156.254:34367(127.27.156.254:34367)] handling channelInactive
21:03:59.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.864820 28275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 28614
21:03:59.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.878484 29417 catalog_manager.cc:1310] Loaded cluster ID: 7c1de3e32ddb4085aac5187ca386de05
21:03:59.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.878773 29417 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: loading cluster ID for follower catalog manager: success
21:03:59.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 14 21:03:59 dist-test-slave-q92b krb5kdc[28287](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1794690238, etypes {rep=17 tkt=17 ses=17}, kudu/127.27.156.253@KRBTEST.COM for kudu/127.27.156.254@KRBTEST.COM
21:03:59.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.902931 29417 catalog_manager.cc:1625] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: acquiring CA information for follower catalog manager: success
21:03:59.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.927403 28275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 28766
21:03:59.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.961385 29417 catalog_manager.cc:1653] T 00000000000000000000000000000000 P b1a43974a60e4addb0042d681f0649e4: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:03:59.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:03:59.984031 28275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 28914
21:04:00.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:04:00.014041 28275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 29064
21:04:00.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:04:00.051095 28275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 29255
21:04:00.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:04:00.085886 28275 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 29146
21:04:01.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-14T21:04:01Z chronyd exiting
21:04:02.384 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:04:02.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:04:02.386742 27552 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 27852
21:04:02.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:04:02.416473 27552 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 27993
21:04:02.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:04:02.444814 27552 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 28130
21:04:02.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:04:02.469616 27552 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 27565
21:04:02.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:04:02.498708 27552 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 27635
21:04:02.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261114 21:04:02.539060 27552 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskerlxO1/build/tsan/bin/kudu with pid 27712
21:04:03.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-14T21:04:03Z chronyd exiting
21:04:04.721 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.27.156.254:34367,127.27.156.252:44355,127.27.156.253:44373). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.27.156.254:34367,127.27.156.252:44355,127.27.156.253:44373). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:291)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:345)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more