16:14:05.138 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
16:14:05.149 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskwnt6fz/test-tmp
16:14:05.155 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:14:05.158 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskwnt6fz/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:14:05.351 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
16:14:05.552 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:14:05.553 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:14:05.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:05.558486 11749 mini_oidc.cc:150] Starting JWKS server
16:14:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:05.562670 11749 webserver.cc:577] Webserver started at http://127.0.0.1:44437/ using document root <none> and password file <none>
16:14:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:05.563261 11749 mini_oidc.cc:186] Starting OIDC Discovery server
16:14:05.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:05.565317 11749 webserver.cc:577] Webserver started at http://127.0.0.1:41475/ using document root <none> and password file <none>
16:14:05.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-19T16:14:05Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
16:14:05.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-19T16:14:05Z Disabled control of system clock
16:14:05.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:05.627405 11749 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:05.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:05.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/wal
16:14:05.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/data
16:14:05.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:05.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:05.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:05.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:05.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:05.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:05.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/logs
16:14:05.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/data/info.pb
16:14:05.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:05.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:05.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:05.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:05.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:05.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:05.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.126:44665
16:14:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.126
16:14:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:05.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:05.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:05.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:05.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:05.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:05.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:05.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44437/jwks.json with env {}
16:14:05.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:05.939395 11761 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:05.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:05.939957 11761 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:05.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:05.940616 11761 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:05.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:05.977757 11761 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:05.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:05.978051 11761 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:05.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:05.978261 11761 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:05.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:05.978490 11761 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:05.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:05.978772 11761 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:05.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:05.978927 11761 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44437/jwks.json
16:14:06.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:06.017246 11761 master_runner.cc:387] Master server non-default flags:
16:14:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/logs
16:14:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:06.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:06.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/data
16:14:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/wal
16:14:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:06.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.126:44665
16:14:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:06.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44437/jwks.json
16:14:06.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:06.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:06.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/data/info.pb
16:14:06.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.126
16:14:06.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:06.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:06.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:06.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:06.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:06.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:06.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:06.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:06.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:06.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:06.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:06.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:06.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:06.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:06.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:06.018764 11761 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:06.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:06.020473 11761 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:06.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:06.032267 11767 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:07.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:06.036180 11770 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:06.034955 11768 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:07.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.317517 11769 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1280 milliseconds
16:14:07.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.317631 11761 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:07.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.319074 11761 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.321805 11761 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:07.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.323233 11761 hybrid_clock.cc:648] HybridClock initialized: now 1795104847323191 us; error 70 us; skew 500 ppm
16:14:07.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.324396 11761 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:07.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.332261 11761 webserver.cc:577] Webserver started at http://127.11.121.126:44227/ using document root <none> and password file <none>
16:14:07.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.333338 11761 fs_manager.cc:362] Metadata directory not provided
16:14:07.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.333654 11761 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:07.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.334298 11761 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.338995 11761 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/data/instance:
16:14:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2ecc778178b4d8fb358f19b804b465d"
16:14:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:07 on dist-test-slave-4vz6"
16:14:07.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.340482 11761 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/wal/instance:
16:14:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2ecc778178b4d8fb358f19b804b465d"
16:14:07.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:07 on dist-test-slave-4vz6"
16:14:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.348115 11761 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
16:14:07.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.354193 11777 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:07.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.355546 11761 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
16:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.355993 11761 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/wal
16:14:07.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b2ecc778178b4d8fb358f19b804b465d"
16:14:07.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:07 on dist-test-slave-4vz6"
16:14:07.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.356454 11761 fs_report.cc:389] FS layout report
16:14:07.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:07.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/wal
16:14:07.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/wal
16:14:07.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/data/data
16:14:07.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:07.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:07.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:07.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:07.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:07.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:07.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:07.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:07.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:07.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:07.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:07.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.406832 11761 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:07.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.408561 11761 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:07.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.409063 11761 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:07.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.479784 11761 rpc_server.cc:307] RPC server started. Bound to: 127.11.121.126:44665
16:14:07.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.479907 11828 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.121.126:44665 every 8 connection(s)
16:14:07.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.482846 11761 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/data/info.pb
16:14:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.488633 11829 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:14:07.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.490573 11749 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 11761
16:14:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.491042 11749 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-0/wal/instance
16:14:07.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.494585 11749 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:07.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:07.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/wal
16:14:07.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/data
16:14:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:07.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/logs
16:14:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/data/info.pb
16:14:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:07.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.125:42099
16:14:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.125
16:14:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:07.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:07.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:07.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44437/jwks.json with env {}
16:14:07.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.501590 11829 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } has no permanent_uuid. Determining permanent_uuid...
16:14:07.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.529754 11829 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } has no permanent_uuid. Determining permanent_uuid...
16:14:07.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.532526 11781 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.121.125:42099: connect: Connection refused (error 111)
16:14:07.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.537165 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.125:42099: Network error: Client connection negotiation failed: client connection to 127.11.121.125:42099: connect: Connection refused (error 111)
16:14:07.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.586766 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } attempt: 1
16:14:07.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.591403 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.125:42099: Network error: Client connection negotiation failed: client connection to 127.11.121.125:42099: connect: Connection refused (error 111)
16:14:07.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.659825 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } attempt: 2
16:14:07.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.665190 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.125:42099: Network error: Client connection negotiation failed: client connection to 127.11.121.125:42099: connect: Connection refused (error 111)
16:14:07.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.756879 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } attempt: 3
16:14:07.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.762168 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.125:42099: Network error: Client connection negotiation failed: client connection to 127.11.121.125:42099: connect: Connection refused (error 111)
16:14:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.818717 11831 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.819263 11831 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:07.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.819762 11831 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:07.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.858466 11831 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:07.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.858753 11831 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:07.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.858959 11831 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:07.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.859160 11831 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:07.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.859422 11831 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:07.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.859612 11831 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44437/jwks.json
16:14:07.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.897089 11831 master_runner.cc:387] Master server non-default flags:
16:14:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/logs
16:14:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/data
16:14:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/wal
16:14:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.125:42099
16:14:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44437/jwks.json
16:14:07.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:07.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:07.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/data/info.pb
16:14:07.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.125
16:14:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:07.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:07.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:07.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:07.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:07.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:07.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.898470 11831 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:07.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.900290 11831 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:07.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:07.905586 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } attempt: 4
16:14:07.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.911258 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.125:42099: Network error: Client connection negotiation failed: client connection to 127.11.121.125:42099: connect: Connection refused (error 111)
16:14:07.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.912549 11840 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:08.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:08.210785 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } attempt: 5
16:14:08.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:08.221764 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.125:42099: Network error: Client connection negotiation failed: client connection to 127.11.121.125:42099: connect: Connection refused (error 111)
16:14:08.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:08.769335 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } attempt: 6
16:14:08.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:08.801911 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.125:42099: Network error: Client connection negotiation failed: client connection to 127.11.121.125:42099: connect: Connection refused (error 111)
16:14:09.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:07.914237 11841 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.197014 11831 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.283s user 0.419s sys 0.853s
16:14:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.197438 11831 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.284s user 0.423s sys 0.853s
16:14:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.198419 11842 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1277 milliseconds
16:14:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.199322 11843 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.199369 11831 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:09.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.200811 11831 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:09.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.203521 11831 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:09.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.204897 11831 hybrid_clock.cc:648] HybridClock initialized: now 1795104849204826 us; error 71 us; skew 500 ppm
16:14:09.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.205912 11831 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:09.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.215567 11831 webserver.cc:577] Webserver started at http://127.11.121.125:43825/ using document root <none> and password file <none>
16:14:09.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.216707 11831 fs_manager.cc:362] Metadata directory not provided
16:14:09.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.216979 11831 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:09.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.217777 11831 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:09.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.223017 11831 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/data/instance:
16:14:09.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83a396980237498d9c599a4b6d48cdb6"
16:14:09.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:09 on dist-test-slave-4vz6"
16:14:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.224378 11831 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/wal/instance:
16:14:09.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83a396980237498d9c599a4b6d48cdb6"
16:14:09.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:09 on dist-test-slave-4vz6"
16:14:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.233181 11831 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.010s sys 0.000s
16:14:09.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.239678 11852 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:09.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.241117 11831 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.003s
16:14:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.241578 11831 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/wal
16:14:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83a396980237498d9c599a4b6d48cdb6"
16:14:09.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:09 on dist-test-slave-4vz6"
16:14:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.242012 11831 fs_report.cc:389] FS layout report
16:14:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/wal
16:14:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/wal
16:14:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/data/data
16:14:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:09.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:09.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:09.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:09.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.319546 11831 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.320960 11831 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:09.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.321444 11831 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:09.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.408334 11831 rpc_server.cc:307] RPC server started. Bound to: 127.11.121.125:42099
16:14:09.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.408416 11903 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.121.125:42099 every 8 connection(s)
16:14:09.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.411356 11831 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/data/info.pb
16:14:09.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.417105 11904 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:14:09.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.419422 11749 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 11831
16:14:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.419864 11749 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-1/wal/instance
16:14:09.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.422856 11749 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:09.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:09.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/wal
16:14:09.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/data
16:14:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/logs
16:14:09.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/data/info.pb
16:14:09.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:09.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:09.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:09.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:09.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:09.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:09.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:09.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.124:40765
16:14:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.124
16:14:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:09.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:09.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:09.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:09.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:09.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:09.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:09.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44437/jwks.json with env {}
16:14:09.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.436205 11904 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } has no permanent_uuid. Determining permanent_uuid...
16:14:09.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.460927 11904 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } has no permanent_uuid. Determining permanent_uuid...
16:14:09.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.477418 11904 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } has no permanent_uuid. Determining permanent_uuid...
16:14:09.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.480566 11854 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:09.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.488561 11904 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:09.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.538153 11904 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 1
16:14:09.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.543284 11904 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:09.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.611688 11904 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 2
16:14:09.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.616436 11904 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:09.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.707834 11904 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 3
16:14:09.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.712535 11904 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:09.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.751658 11906 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:09.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.752410 11906 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:09.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.753120 11906 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.786535 11906 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.786952 11906 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.787303 11906 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:09.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.787698 11906 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.788076 11906 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:09.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.788355 11906 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:44437/jwks.json
16:14:09.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.828151 11906 master_runner.cc:387] Master server non-default flags:
16:14:09.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/logs
16:14:09.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:09.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:09.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:09.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:09.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/data
16:14:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/wal
16:14:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:09.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.124:40765
16:14:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:44437/jwks.json
16:14:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/data/info.pb
16:14:09.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.124
16:14:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:09.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:09.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.829885 11906 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:09.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.832188 11906 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:09.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.843847 11916 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:09.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.855930 11904 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 4
16:14:09.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.860743 11904 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:09.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.862538 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } attempt: 7
16:14:09.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.875980 11829 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } has no permanent_uuid. Determining permanent_uuid...
16:14:09.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.885149 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:09.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:09.943660 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 1
16:14:09.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.950003 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:10.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:10.031430 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 2
16:14:10.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:10.037720 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:10.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:10.123138 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 3
16:14:10.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:10.129516 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:10.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:10.160171 11904 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 5
16:14:10.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:10.165833 11904 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:10.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:10.269989 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 4
16:14:10.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:10.275863 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:10.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:10.559309 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 5
16:14:10.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:10.565739 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:10.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:10.713254 11904 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 6
16:14:10.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:10.722213 11904 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:11.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:09.846302 11917 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:11.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:11.065119 11918 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:14:11.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:11.066677 11919 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:11.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.066701 11906 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:11.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.070057 11906 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:11.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.072624 11906 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:11.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.073956 11906 hybrid_clock.cc:648] HybridClock initialized: now 1795104851073929 us; error 59 us; skew 500 ppm
16:14:11.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.074795 11906 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:11.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.080870 11906 webserver.cc:577] Webserver started at http://127.11.121.124:44541/ using document root <none> and password file <none>
16:14:11.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.081817 11906 fs_manager.cc:362] Metadata directory not provided
16:14:11.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.082001 11906 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:11.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.082386 11906 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.086544 11906 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/data/instance:
16:14:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a15edc36c8e34139ac9091d8bacfbbe1"
16:14:11.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:11 on dist-test-slave-4vz6"
16:14:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.087590 11906 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/wal/instance:
16:14:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a15edc36c8e34139ac9091d8bacfbbe1"
16:14:11.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:11 on dist-test-slave-4vz6"
16:14:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.095933 11906 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.009s sys 0.000s
16:14:11.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.102483 11930 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:11.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.104228 11906 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
16:14:11.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.104547 11906 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/wal
16:14:11.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a15edc36c8e34139ac9091d8bacfbbe1"
16:14:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:11 on dist-test-slave-4vz6"
16:14:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.104837 11906 fs_report.cc:389] FS layout report
16:14:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/wal
16:14:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/wal
16:14:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/data/data
16:14:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:11.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:11.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:11.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.118177 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 6
16:14:11.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:11.125038 11829 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.11.121.124:40765: Network error: Client connection negotiation failed: client connection to 127.11.121.124:40765: connect: Connection refused (error 111)
16:14:11.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.160221 11906 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:11.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.161737 11906 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:11.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.162141 11906 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.233598 11906 rpc_server.cc:307] RPC server started. Bound to: 127.11.121.124:40765
16:14:11.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.233691 11983 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.121.124:40765 every 8 connection(s)
16:14:11.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.236654 11906 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/data/info.pb
16:14:11.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.242825 11984 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:14:11.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.245440 11749 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 11906
16:14:11.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.245853 11749 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/master-2/wal/instance
16:14:11.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.256350 11984 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } has no permanent_uuid. Determining permanent_uuid...
16:14:11.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.281512 11984 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } has no permanent_uuid. Determining permanent_uuid...
16:14:11.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.291544 11984 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } has no permanent_uuid. Determining permanent_uuid...
16:14:11.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.310220 11984 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1: Bootstrap starting.
16:14:11.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.317276 11984 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1: Neither blocks nor log segments found. Creating new log.
16:14:11.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.319275 11984 log.cc:826] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1: Log is configured to *not* fsync() on all Append() calls
16:14:11.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.323410 11984 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1: No bootstrap required, opened a new log
16:14:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.343484 11984 raft_consensus.cc:360] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } }
16:14:11.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.344448 11984 raft_consensus.cc:386] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:14:11.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.344688 11984 raft_consensus.cc:748] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a15edc36c8e34139ac9091d8bacfbbe1, State: Initialized, Role: FOLLOWER
16:14:11.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.345924 11984 consensus_queue.cc:261] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } }
16:14:11.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.349977 11993 sys_catalog.cc:455] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } } }
16:14:11.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.350854 11993 sys_catalog.cc:458] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [sys.catalog]: This master's current role is: FOLLOWER
16:14:11.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.351761 11984 sys_catalog.cc:565] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [sys.catalog]: configured and running, proceeding with master startup.
16:14:11.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:11.379675 12004 catalog_manager.cc:1609] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:14:11.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:11.380643 12004 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:14:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.373912 11906 master_runner.cc:187] Error getting master registration for 127.11.121.126:44665: OK, instance_id { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" instance_seqno: 1795104847461304 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:14:11.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.397615 11906 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:14:11.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:11.417243 11938 tablet.cc:2554] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:14:11.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.506414 11993 raft_consensus.cc:499] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:14:11.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.506865 11993 raft_consensus.cc:521] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } }
16:14:11.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.508966 11993 leader_election.cc:290] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers b2ecc778178b4d8fb358f19b804b465d (127.11.121.126:44665), 83a396980237498d9c599a4b6d48cdb6 (127.11.121.125:42099)
16:14:11.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.782810 11904 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 7
16:14:11.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.805681 11904 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6: Bootstrap starting.
16:14:11.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.813858 11904 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6: Neither blocks nor log segments found. Creating new log.
16:14:11.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.816810 11904 log.cc:826] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6: Log is configured to *not* fsync() on all Append() calls
16:14:11.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.824801 11904 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6: No bootstrap required, opened a new log
16:14:11.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.854883 11904 raft_consensus.cc:360] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } }
16:14:11.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.857862 11904 raft_consensus.cc:386] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:14:11.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.858462 11904 raft_consensus.cc:748] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 83a396980237498d9c599a4b6d48cdb6, State: Initialized, Role: FOLLOWER
16:14:11.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.859627 11904 consensus_queue.cc:261] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } }
16:14:11.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.863327 12008 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } } }
16:14:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.865114 11904 sys_catalog.cc:565] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [sys.catalog]: configured and running, proceeding with master startup.
16:14:11.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.865031 12008 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [sys.catalog]: This master's current role is: FOLLOWER
16:14:11.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.865294 11879 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83a396980237498d9c599a4b6d48cdb6" is_pre_election: true
16:14:11.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.868471 11879 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a15edc36c8e34139ac9091d8bacfbbe1 in term 0.
16:14:11.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.870162 11936 leader_election.cc:304] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 83a396980237498d9c599a4b6d48cdb6, a15edc36c8e34139ac9091d8bacfbbe1; no voters:
16:14:11.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.871147 11993 raft_consensus.cc:2826] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:14:11.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.871482 11993 raft_consensus.cc:499] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:14:11.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.871801 11993 raft_consensus.cc:3096] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [term 0 FOLLOWER]: Advancing to term 1
16:14:11.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.879117 11993 raft_consensus.cc:521] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } }
16:14:11.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.882345 11993 leader_election.cc:290] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [CANDIDATE]: Term 1 election: Requested vote from peers b2ecc778178b4d8fb358f19b804b465d (127.11.121.126:44665), 83a396980237498d9c599a4b6d48cdb6 (127.11.121.125:42099)
16:14:11.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.883817 11879 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "83a396980237498d9c599a4b6d48cdb6"
16:14:11.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.884547 11879 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [term 0 FOLLOWER]: Advancing to term 1
16:14:11.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.902076 11879 raft_consensus.cc:2490] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a15edc36c8e34139ac9091d8bacfbbe1 in term 1.
16:14:11.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.903769 11936 leader_election.cc:304] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 83a396980237498d9c599a4b6d48cdb6, a15edc36c8e34139ac9091d8bacfbbe1; no voters:
16:14:11.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.904798 11993 raft_consensus.cc:2826] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [term 1 FOLLOWER]: Leader election won for term 1
16:14:11.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.909854 11993 raft_consensus.cc:705] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [term 1 LEADER]: Becoming Leader. State: Replica: a15edc36c8e34139ac9091d8bacfbbe1, State: Running, Role: LEADER
16:14:11.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.910835 11993 consensus_queue.cc:238] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } }
16:14:11.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.925495 12017 sys_catalog.cc:455] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a15edc36c8e34139ac9091d8bacfbbe1. Latest consensus state: current_term: 1 leader_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } } }
16:14:11.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.927146 12017 sys_catalog.cc:458] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [sys.catalog]: This master's current role is: LEADER
16:14:11.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.930732 12021 consensus_queue.cc:1049] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
16:14:11.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.935391 12022 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
16:14:11.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.952414 12008 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a15edc36c8e34139ac9091d8bacfbbe1. Latest consensus state: current_term: 1 leader_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } } }
16:14:11.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.953153 12008 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [sys.catalog]: This master's current role is: FOLLOWER
16:14:11.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.957729 12022 catalog_manager.cc:1535] Initializing Kudu cluster ID...
16:14:11.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.964098 12021 sys_catalog.cc:455] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } } }
16:14:11.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.964819 12021 sys_catalog.cc:458] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [sys.catalog]: This master's current role is: LEADER
16:14:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.969192 12008 sys_catalog.cc:455] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } } }
16:14:11.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.969969 12008 sys_catalog.cc:458] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6 [sys.catalog]: This master's current role is: FOLLOWER
16:14:11.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:11.992413 12024 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:14:11.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:11.992851 12024 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:14:12.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:11.959558 11831 master_runner.cc:187] Error getting master registration for 127.11.121.126:44665: OK, instance_id { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" instance_seqno: 1795104847461304 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:14:12.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.021664 11831 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.026319 12022 catalog_manager.cc:1398] Generated new cluster ID: 68a24201e6144aa6a7d51a18bcd5abe9
16:14:12.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.026644 12022 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
16:14:12.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.092377 12022 catalog_manager.cc:1421] Generated new certificate authority record
16:14:12.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.096655 12022 catalog_manager.cc:1555] Loading token signing keys...
16:14:12.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.158480 11829 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } attempt: 7
16:14:12.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.180778 12022 catalog_manager.cc:6237] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1: Generated new TSK 0
16:14:12.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.182312 12022 catalog_manager.cc:1565] Initializing in-progress tserver states...
16:14:12.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.205255 11749 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/wal
16:14:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/data
16:14:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:12.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/logs
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/data/info.pb
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:12.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.65:0
16:14:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.121.65
16:14:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.65
16:14:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:12.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:14:12.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.218449 11829 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d: Bootstrap starting.
16:14:12.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.226490 11829 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d: Neither blocks nor log segments found. Creating new log.
16:14:12.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.243260 11829 log.cc:826] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d: Log is configured to *not* fsync() on all Append() calls
16:14:12.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.265126 11829 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d: No bootstrap required, opened a new log
16:14:12.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.318600 11829 raft_consensus.cc:360] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } }
16:14:12.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.319885 11829 raft_consensus.cc:386] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:14:12.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.320380 11829 raft_consensus.cc:748] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b2ecc778178b4d8fb358f19b804b465d, State: Initialized, Role: FOLLOWER
16:14:12.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.321642 11829 consensus_queue.cc:261] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } }
16:14:12.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.338937 12033 sys_catalog.cc:455] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } } }
16:14:12.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.342945 12033 sys_catalog.cc:458] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [sys.catalog]: This master's current role is: FOLLOWER
16:14:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.344344 11802 raft_consensus.cc:3096] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [term 0 FOLLOWER]: Advancing to term 1
16:14:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.343422 11804 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b2ecc778178b4d8fb358f19b804b465d" is_pre_election: true
16:14:12.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.342804 11803 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "b2ecc778178b4d8fb358f19b804b465d"
16:14:12.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.342628 11829 sys_catalog.cc:565] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [sys.catalog]: configured and running, proceeding with master startup.
16:14:12.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.356065 12021 consensus_queue.cc:1049] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:14:12.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.429517 12033 sys_catalog.cc:455] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [sys.catalog]: SysCatalogTable state changed. Reason: New leader a15edc36c8e34139ac9091d8bacfbbe1. Latest consensus state: current_term: 1 leader_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } } }
16:14:12.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.437477 12033 sys_catalog.cc:458] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [sys.catalog]: This master's current role is: FOLLOWER
16:14:12.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.442981 12041 sys_catalog.cc:455] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } } }
16:14:12.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.444152 12041 sys_catalog.cc:458] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d [sys.catalog]: This master's current role is: FOLLOWER
16:14:12.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.463182 12050 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:14:12.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.812273 12052 sys_catalog.cc:455] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b2ecc778178b4d8fb358f19b804b465d" member_type: VOTER last_known_addr { host: "127.11.121.126" port: 44665 } } peers { permanent_uuid: "83a396980237498d9c599a4b6d48cdb6" member_type: VOTER last_known_addr { host: "127.11.121.125" port: 42099 } } peers { permanent_uuid: "a15edc36c8e34139ac9091d8bacfbbe1" member_type: VOTER last_known_addr { host: "127.11.121.124" port: 40765 } } }
16:14:12.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.815846 12052 sys_catalog.cc:458] T 00000000000000000000000000000000 P a15edc36c8e34139ac9091d8bacfbbe1 [sys.catalog]: This master's current role is: LEADER
16:14:12.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:12.884687 12031 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:12.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:12.885406 12031 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:12.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:12.886186 12031 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:12.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:12.948768 12031 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:12.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:12.950297 12031 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.121.65
16:14:12.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.999058 12024 catalog_manager.cc:1310] Loaded cluster ID: 68a24201e6144aa6a7d51a18bcd5abe9
16:14:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:12.999802 12024 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6: loading cluster ID for follower catalog manager: success
16:14:13.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:13.008329 12024 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6: acquiring CA information for follower catalog manager: success
16:14:13.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:13.016865 12024 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 83a396980237498d9c599a4b6d48cdb6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:14:13.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:13.019392 12031 tablet_server_runner.cc:78] Tablet server non-default flags:
16:14:13.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/logs
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/data
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/wal
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.65:0
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:13.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/data/info.pb
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.65
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.121.65
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:14:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:13.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:13.023221 12031 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:13.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:13.026355 12031 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:13.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:13.043660 12059 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:13.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:13.504228 12050 catalog_manager.cc:1310] Loaded cluster ID: 68a24201e6144aa6a7d51a18bcd5abe9
16:14:13.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:13.522855 12050 catalog_manager.cc:1603] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d: loading cluster ID for follower catalog manager: success
16:14:13.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:13.577595 12050 catalog_manager.cc:1625] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d: acquiring CA information for follower catalog manager: success
16:14:13.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:13.611503 12050 catalog_manager.cc:1653] T 00000000000000000000000000000000 P b2ecc778178b4d8fb358f19b804b465d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:14:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:13.051606 12031 server_base.cc:1061] running on GCE node
16:14:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:13.052700 12062 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:14.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:13.050066 12060 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:14.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.510493 12031 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:14.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.517848 12031 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:14.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.521718 12031 hybrid_clock.cc:648] HybridClock initialized: now 1795104854521668 us; error 54 us; skew 500 ppm
16:14:14.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.522780 12031 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:14.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.530453 12031 webserver.cc:577] Webserver started at http://127.11.121.65:39413/ using document root <none> and password file <none>
16:14:14.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.531644 12031 fs_manager.cc:362] Metadata directory not provided
16:14:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.531913 12031 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:14.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.532469 12031 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:14.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.538901 12031 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/data/instance:
16:14:14.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "692a2688eeca406db8bee0e05b26839b"
16:14:14.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:14 on dist-test-slave-4vz6"
16:14:14.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.540442 12031 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/wal/instance:
16:14:14.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "692a2688eeca406db8bee0e05b26839b"
16:14:14.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:14 on dist-test-slave-4vz6"
16:14:14.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.549851 12031 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
16:14:14.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.557538 12070 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.558791 12031 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
16:14:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.559079 12031 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/wal
16:14:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "692a2688eeca406db8bee0e05b26839b"
16:14:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:14 on dist-test-slave-4vz6"
16:14:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.559365 12031 fs_report.cc:389] FS layout report
16:14:14.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/wal
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/wal
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/data/data
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:14.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:14.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.610656 12031 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.612092 12031 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:14.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.612491 12031 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:14.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.616015 12031 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:14:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.620775 12031 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:14:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.621007 12031 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:14:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.621316 12031 ts_tablet_manager.cc:662] Registered 0 tablets
16:14:14.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.621582 12031 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
16:14:14.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.794312 12031 rpc_server.cc:307] RPC server started. Bound to: 127.11.121.65:34559
16:14:14.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.794406 12182 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.121.65:34559 every 8 connection(s)
16:14:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.797490 12031 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/data/info.pb
16:14:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.804557 11749 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 12031
16:14:14.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.804996 11749 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-0/wal/instance
16:14:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.834743 11749 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/wal
16:14:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/data
16:14:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:14.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/logs
16:14:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/data/info.pb
16:14:14.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:14.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:14.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:14.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:14.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:14.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:14.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:14.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:14.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:14:14.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:14.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.66:0
16:14:14.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.121.66
16:14:14.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.66
16:14:14.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:14.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:14.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:14.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:14.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:14.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:14:14.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.869055 12183 heartbeater.cc:345] Connected to a master server at 127.11.121.124:40765
16:14:14.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.869587 12183 heartbeater.cc:462] Registering TS with master...
16:14:14.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.870924 12183 heartbeater.cc:508] Master 127.11.121.124:40765 requested a full tablet report, sending...
16:14:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.872875 12184 heartbeater.cc:345] Connected to a master server at 127.11.121.125:42099
16:14:14.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.873222 12184 heartbeater.cc:462] Registering TS with master...
16:14:14.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.874104 12184 heartbeater.cc:508] Master 127.11.121.125:42099 requested a full tablet report, sending...
16:14:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.874398 11949 ts_manager.cc:210] Registered new tserver with Master: 692a2688eeca406db8bee0e05b26839b (127.11.121.65:34559)
16:14:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.877095 11949 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.11.121.65:39897
16:14:14.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.877425 11869 ts_manager.cc:210] Registered new tserver with Master: 692a2688eeca406db8bee0e05b26839b (127.11.121.65:34559)
16:14:14.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.883636 12185 heartbeater.cc:345] Connected to a master server at 127.11.121.126:44665
16:14:14.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.883958 12185 heartbeater.cc:462] Registering TS with master...
16:14:14.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.884588 12185 heartbeater.cc:508] Master 127.11.121.126:44665 requested a full tablet report, sending...
16:14:14.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:14.888579 11794 ts_manager.cc:210] Registered new tserver with Master: 692a2688eeca406db8bee0e05b26839b (127.11.121.65:34559)
16:14:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:15.197579 12191 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:15.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:15.198122 12191 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:15.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:15.198720 12191 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:15.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:15.234277 12191 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:15.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:15.235668 12191 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.121.66
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:15.288463 12191 tablet_server_runner.cc:78] Tablet server non-default flags:
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/logs
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/data
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/wal
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:15.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.66:0
16:14:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/data/info.pb
16:14:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.66
16:14:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:15.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.121.66
16:14:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:14:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:15.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:15.289902 12191 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:15.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:15.291509 12191 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:15.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:15.307225 12199 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:15.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:15.881776 12183 heartbeater.cc:500] Master 127.11.121.124:40765 was elected leader, sending a full tablet report...
16:14:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:16.710883 12198 debug-util.cc:398] Leaking SignalData structure 0x7b08000382a0 after lost signal to thread 12191
16:14:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.142916 12198 kernel_stack_watchdog.cc:198] Thread 12191 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 399ms:
16:14:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:14:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:14:17.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:14:17.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:14:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:15.309078 12200 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.144840 12191 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.836s user 0.546s sys 1.113s
16:14:17.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.145238 12191 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.837s user 0.546s sys 1.114s
16:14:17.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.145841 12202 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:17.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.146121 12201 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1835 milliseconds
16:14:17.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.146551 12191 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:17.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.151582 12191 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:17.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.157050 12191 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:17.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.158699 12191 hybrid_clock.cc:648] HybridClock initialized: now 1795104857158612 us; error 92 us; skew 500 ppm
16:14:17.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.159994 12191 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:17.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.168326 12191 webserver.cc:577] Webserver started at http://127.11.121.66:39457/ using document root <none> and password file <none>
16:14:17.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.169561 12191 fs_manager.cc:362] Metadata directory not provided
16:14:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.169863 12191 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:17.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.170451 12191 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.176637 12191 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/data/instance:
16:14:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "edaf336e3a314af6bf1c50ea91942540"
16:14:17.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:17 on dist-test-slave-4vz6"
16:14:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.178210 12191 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/wal/instance:
16:14:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "edaf336e3a314af6bf1c50ea91942540"
16:14:17.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:17 on dist-test-slave-4vz6"
16:14:17.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.189764 12191 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.005s sys 0.005s
16:14:17.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.199600 12211 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:17.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.203578 12191 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.005s sys 0.000s
16:14:17.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.204156 12191 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/wal
16:14:17.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "edaf336e3a314af6bf1c50ea91942540"
16:14:17.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:17 on dist-test-slave-4vz6"
16:14:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.204698 12191 fs_report.cc:389] FS layout report
16:14:17.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:17.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/wal
16:14:17.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/wal
16:14:17.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/data/data
16:14:17.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:17.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:17.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.284096 12191 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:17.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.285899 12191 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:17.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.286398 12191 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:17.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.289100 12191 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:14:17.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.294389 12191 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:14:17.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.294627 12191 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:14:17.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.294960 12191 ts_tablet_manager.cc:662] Registered 0 tablets
16:14:17.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.295184 12191 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:14:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.430625 12191 rpc_server.cc:307] RPC server started. Bound to: 127.11.121.66:37311
16:14:17.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.430718 12323 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.121.66:37311 every 8 connection(s)
16:14:17.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.434237 12191 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/data/info.pb
16:14:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.438483 11749 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 12191
16:14:17.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.439101 11749 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-1/wal/instance
16:14:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.456902 11749 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/wal
16:14:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/data
16:14:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:17.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/logs
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/data/info.pb
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.67:0
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.121.67
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.67
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:17.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:14:17.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.483163 12324 heartbeater.cc:345] Connected to a master server at 127.11.121.124:40765
16:14:17.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.483657 12324 heartbeater.cc:462] Registering TS with master...
16:14:17.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.485088 12324 heartbeater.cc:508] Master 127.11.121.124:40765 requested a full tablet report, sending...
16:14:17.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.487902 12325 heartbeater.cc:345] Connected to a master server at 127.11.121.125:42099
16:14:17.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.488291 12325 heartbeater.cc:462] Registering TS with master...
16:14:17.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.488266 11949 ts_manager.cc:210] Registered new tserver with Master: edaf336e3a314af6bf1c50ea91942540 (127.11.121.66:37311)
16:14:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.489279 12325 heartbeater.cc:508] Master 127.11.121.125:42099 requested a full tablet report, sending...
16:14:17.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.490024 11949 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.11.121.66:50959
16:14:17.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.492549 11869 ts_manager.cc:210] Registered new tserver with Master: edaf336e3a314af6bf1c50ea91942540 (127.11.121.66:37311)
16:14:17.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.496465 12326 heartbeater.cc:345] Connected to a master server at 127.11.121.126:44665
16:14:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.496843 12326 heartbeater.cc:462] Registering TS with master...
16:14:17.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.497607 12326 heartbeater.cc:508] Master 127.11.121.126:44665 requested a full tablet report, sending...
16:14:17.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.500874 11794 ts_manager.cc:210] Registered new tserver with Master: edaf336e3a314af6bf1c50ea91942540 (127.11.121.66:37311)
16:14:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.814878 12331 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.815338 12331 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:17.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.815768 12331 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:17.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.854108 12331 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:17.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.854948 12331 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.121.67
16:14:17.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.895906 12331 tablet_server_runner.cc:78] Tablet server non-default flags:
16:14:17.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/logs
16:14:17.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:17.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:17.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.121.84:39243
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/data
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/wal
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.121.67:0
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/data/info.pb
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.121.67
16:14:17.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.121.126:44665,127.11.121.125:42099,127.11.121.124:40765
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.121.67
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:17.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.897574 12331 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:17.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:17.899312 12331 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:17.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.911968 12341 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:18.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:18.494127 12324 heartbeater.cc:500] Master 127.11.121.124:40765 was elected leader, sending a full tablet report...
16:14:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:19.316779 12340 debug-util.cc:398] Leaking SignalData structure 0x7b08000382a0 after lost signal to thread 12331
16:14:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:19.456640 12331 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.544s user 0.614s sys 0.923s
16:14:19.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:19.457255 12331 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.545s user 0.614s sys 0.923s
16:14:19.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:19.457717 12343 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1543 milliseconds
16:14:19.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:17.913430 12342 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:19.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.458853 12331 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:19.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:19.458928 12344 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:19.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.463925 12331 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:19.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.466293 12331 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:19.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.467669 12331 hybrid_clock.cc:648] HybridClock initialized: now 1795104859467608 us; error 65 us; skew 500 ppm
16:14:19.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.468428 12331 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:19.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.474363 12331 webserver.cc:577] Webserver started at http://127.11.121.67:33409/ using document root <none> and password file <none>
16:14:19.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.475172 12331 fs_manager.cc:362] Metadata directory not provided
16:14:19.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.475350 12331 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:19.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.475723 12331 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:19.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.480243 12331 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/data/instance:
16:14:19.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b855438ebe224891bf160088d502319c"
16:14:19.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:19 on dist-test-slave-4vz6"
16:14:19.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.481271 12331 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/wal/instance:
16:14:19.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b855438ebe224891bf160088d502319c"
16:14:19.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:19 on dist-test-slave-4vz6"
16:14:19.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.488528 12331 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
16:14:19.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.495309 12351 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:19.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.496989 12331 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.006s sys 0.000s
16:14:19.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.497293 12331 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/wal
16:14:19.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b855438ebe224891bf160088d502319c"
16:14:19.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:19 on dist-test-slave-4vz6"
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.497644 12331 fs_report.cc:389] FS layout report
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/wal
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/wal
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/data/data
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:19.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:19.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.547420 12331 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:19.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.548986 12331 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:19.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.549435 12331 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:19.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.551949 12331 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:14:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.556264 12331 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:14:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.556452 12331 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:14:19.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.556638 12331 ts_tablet_manager.cc:662] Registered 0 tablets
16:14:19.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.556763 12331 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:14:19.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.688504 12331 rpc_server.cc:307] RPC server started. Bound to: 127.11.121.67:42011
16:14:19.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.688742 12463 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.121.67:42011 every 8 connection(s)
16:14:19.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.691773 12331 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/data/info.pb
16:14:19.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.699312 11749 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 12331
16:14:19.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.699882 11749 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster679812881331050091/ts-2/wal/instance
16:14:19.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.734549 12464 heartbeater.cc:345] Connected to a master server at 127.11.121.124:40765
16:14:19.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.735071 12464 heartbeater.cc:462] Registering TS with master...
16:14:19.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.735247 12465 heartbeater.cc:345] Connected to a master server at 127.11.121.125:42099
16:14:19.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.735538 12465 heartbeater.cc:462] Registering TS with master...
16:14:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.736195 12466 heartbeater.cc:345] Connected to a master server at 127.11.121.126:44665
16:14:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.736439 12464 heartbeater.cc:508] Master 127.11.121.124:40765 requested a full tablet report, sending...
16:14:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.736543 12466 heartbeater.cc:462] Registering TS with master...
16:14:19.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.736574 12465 heartbeater.cc:508] Master 127.11.121.125:42099 requested a full tablet report, sending...
16:14:19.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.737497 12466 heartbeater.cc:508] Master 127.11.121.126:44665 requested a full tablet report, sending...
16:14:19.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.739100 11949 ts_manager.cc:210] Registered new tserver with Master: b855438ebe224891bf160088d502319c (127.11.121.67:42011)
16:14:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.739966 11794 ts_manager.cc:210] Registered new tserver with Master: b855438ebe224891bf160088d502319c (127.11.121.67:42011)
16:14:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.740072 11869 ts_manager.cc:210] Registered new tserver with Master: b855438ebe224891bf160088d502319c (127.11.121.67:42011)
16:14:19.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.740543 11949 master_service.cc:507] Signed X509 certificate for tserver {username='slave'} at 127.11.121.67:42811
16:14:19.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:19.765671 11749 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
16:14:19.777 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:14:19.778 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
16:14:19.786 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.121.126"
port: 44665
}
bound_http_address {
host: "127.11.121.126"
port: 44227
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.121.125"
port: 42099
}
bound_http_address {
host: "127.11.121.125"
port: 43825
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.121.124"
port: 40765
}
bound_http_address {
host: "127.11.121.124"
port: 44541
}
}
}
16:14:19.789 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:14:19.792 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.11.121.65"
port: 34559
}
bound_http_address {
host: "127.11.121.65"
port: 39413
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.11.121.66"
port: 37311
}
bound_http_address {
host: "127.11.121.66"
port: 39457
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.11.121.67"
port: 42011
}
bound_http_address {
host: "127.11.121.67"
port: 33409
}
}
}
16:14:19.792 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
16:14:19.974 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
16:14:20.031 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
16:14:20.132 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskwnt6fz/test-tmp
16:14:20.133 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:14:20.133 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskwnt6fz/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:14:20.139 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
16:14:20.496 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:20.497 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:14:20.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:20.500865 12477 mini_oidc.cc:150] Starting JWKS server
16:14:20.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:20.507018 12477 webserver.cc:577] Webserver started at http://127.0.0.1:42857/ using document root <none> and password file <none>
16:14:20.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:20.507535 12477 mini_oidc.cc:186] Starting OIDC Discovery server
16:14:20.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:20.509238 12477 webserver.cc:577] Webserver started at http://127.0.0.1:33939/ using document root <none> and password file <none>
16:14:20.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
16:14:20.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/principal' for realm 'KRBTEST.COM',
16:14:20.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
16:14:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:20 dist-test-slave-4vz6 krb5kdc[12489](info): setting up network...
16:14:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
16:14:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:20 dist-test-slave-4vz6 krb5kdc[12489](info): set up 2 sockets
16:14:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:20 dist-test-slave-4vz6 krb5kdc[12489](info): commencing operation
16:14:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
16:14:20.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:20.744732 12464 heartbeater.cc:500] Master 127.11.121.124:40765 was elected leader, sending a full tablet report...
16:14:22.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:22.651927 12477 mini_kdc.cc:121] Time spent starting KDC: real 2.104s user 0.002s sys 0.017s
16:14:22.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
16:14:22.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:14:22.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
16:14:22.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
16:14:22.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:14:22.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
16:14:22.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
16:14:22.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:14:22.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
16:14:22.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:14:22.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/test-user.keytab.
16:14:22.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/test-user.keytab.
16:14:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:22 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104862, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:22.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:14:22.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-19T16:14:22Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
16:14:22.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-19T16:14:22Z Disabled control of system clock
16:14:22.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.47.126@KRBTEST.COM; defaulting to no policy
16:14:22.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:22.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.47.126@KRBTEST.COM" created.
16:14:22.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:22.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:22.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:22.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.47.126@KRBTEST.COM; defaulting to no policy
16:14:22.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:22.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.47.126@KRBTEST.COM" created.
16:14:22.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:22.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:22.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:22.886904 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/logs
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/info.pb
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:22.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.126:33337
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.126
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.126
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:22.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:22.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:22.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:22.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:22.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:22.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:14:22.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:23.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.203168 12506 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:23.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.203794 12506 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:23.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.204391 12506 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:23.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.238740 12506 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:14:23.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.238967 12506 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:23.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.239245 12506 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:23.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.239454 12506 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:23.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.239675 12506 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.239876 12506 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.240026 12506 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42857/jwks.json
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:23.279798 12506 master_runner.cc:387] Master server non-default flags:
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/logs
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:14:23.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.126
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.126:33337
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/info.pb
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.126
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:23.281167 12506 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:23.282433 12506 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:23.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.294638 12512 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:24.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:24.699278 12511 debug-util.cc:398] Leaking SignalData structure 0x7b0800038620 after lost signal to thread 12506
16:14:24.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:24.832196 12511 kernel_stack_watchdog.cc:198] Thread 12506 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 400ms:
16:14:24.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:14:24.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:14:24.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:14:24.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:14:24.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:23.295054 12513 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:24.833901 12506 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.539s user 0.001s sys 0.001s
16:14:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:24.834285 12506 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.540s user 0.001s sys 0.002s
16:14:24.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:24.834354 12514 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1537 milliseconds
16:14:24.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:24.836881 12515 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:24.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.836792 12506 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:24.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.839293 12506 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:24.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.841951 12506 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:24.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.843370 12506 hybrid_clock.cc:648] HybridClock initialized: now 1795104864843336 us; error 39 us; skew 500 ppm
16:14:24.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.844326 12506 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:24.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:24 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104864, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:24.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.850481 12506 init.cc:377] Logged in from keytab as kudu/127.12.47.126@KRBTEST.COM (short username kudu)
16:14:24.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.855361 12506 webserver.cc:577] Webserver started at http://127.12.47.126:37859/ using document root <none> and password file <none>
16:14:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.856293 12506 fs_manager.cc:362] Metadata directory not provided
16:14:24.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.856534 12506 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:24.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.857077 12506 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:24.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.861407 12506 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/instance:
16:14:24.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "127f4583c0704b96b6ee60f15dbd6a5f"
16:14:24.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:24 on dist-test-slave-4vz6"
16:14:24.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.862659 12506 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal/instance:
16:14:24.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "127f4583c0704b96b6ee60f15dbd6a5f"
16:14:24.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:24 on dist-test-slave-4vz6"
16:14:24.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.870217 12506 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
16:14:24.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.876062 12523 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:24.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.877140 12506 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
16:14:24.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.877520 12506 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:24.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "127f4583c0704b96b6ee60f15dbd6a5f"
16:14:24.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:24 on dist-test-slave-4vz6"
16:14:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.877853 12506 fs_report.cc:389] FS layout report
16:14:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/data
16:14:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:24.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:24.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:24.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:24.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:24.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:24.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:24.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:24.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.933194 12506 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:24.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.944628 12506 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:24.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:24.945045 12506 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:25.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.018420 12506 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.126:33337
16:14:25.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.018482 12574 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.126:33337 every 8 connection(s)
16:14:25.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.021096 12506 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/info.pb
16:14:25.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.022607 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 12506
16:14:25.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.023089 12477 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal/instance
16:14:25.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.027755 12575 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:14:25.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.038456 12575 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } has no permanent_uuid. Determining permanent_uuid...
16:14:25.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.47.125@KRBTEST.COM; defaulting to no policy
16:14:25.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:25.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.47.125@KRBTEST.COM" created.
16:14:25.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:25 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104864, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.126@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:25.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:25.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:25.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:25.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.080778 12575 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } has no permanent_uuid. Determining permanent_uuid...
16:14:25.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.083591 12524 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:25.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.47.125@KRBTEST.COM; defaulting to no policy
16:14:25.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.087875 12575 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:25.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.47.125@KRBTEST.COM" created.
16:14:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:25.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.114781 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/logs
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/info.pb
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:25.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.125:38177
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.125
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.125
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:14:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:25.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.137434 12575 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } attempt: 1
16:14:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.142302 12575 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:25.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.210757 12575 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } attempt: 2
16:14:25.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.215571 12575 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:25.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.307078 12575 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } attempt: 3
16:14:25.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.311862 12575 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:25.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.455372 12575 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } attempt: 4
16:14:25.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.459867 12575 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:25.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.461700 12584 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:25.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.462230 12584 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:25.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.462862 12584 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:25.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.497946 12584 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:14:25.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.498230 12584 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:25.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.498674 12584 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:25.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.499025 12584 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:25.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.499418 12584 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.499779 12584 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:25.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.500068 12584 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42857/jwks.json
16:14:25.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.542799 12584 master_runner.cc:387] Master server non-default flags:
16:14:25.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/logs
16:14:25.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:25.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:25.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:25.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:25.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:25.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:25.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.125
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.125:38177
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:25.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/info.pb
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.125
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:25.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.544437 12584 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:25.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.545917 12584 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:25.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.557817 12592 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:25.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:25.759349 12575 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } attempt: 5
16:14:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.770640 12575 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:26.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:26.318188 12575 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } attempt: 6
16:14:26.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:26.356029 12575 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:27.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:25.559137 12593 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:27.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.380018 12594 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1820 milliseconds
16:14:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:26.962940 12591 debug-util.cc:398] Leaking SignalData structure 0x7b0800038620 after lost signal to thread 12584
16:14:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.381724 12591 kernel_stack_watchdog.cc:198] Thread 12584 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 402ms:
16:14:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:14:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:14:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:14:27.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:14:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.385010 12584 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.827s user 0.662s sys 1.091s
16:14:27.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.385635 12584 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.828s user 0.662s sys 1.092s
16:14:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.387192 12595 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:27.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.387197 12584 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:27.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.388594 12584 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:27.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.391721 12584 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:27.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.393131 12584 hybrid_clock.cc:648] HybridClock initialized: now 1795104867393084 us; error 53 us; skew 500 ppm
16:14:27.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.394197 12584 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:27.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:27 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104867, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:27.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.400583 12584 init.cc:377] Logged in from keytab as kudu/127.12.47.125@KRBTEST.COM (short username kudu)
16:14:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.405750 12584 webserver.cc:577] Webserver started at http://127.12.47.125:36133/ using document root <none> and password file <none>
16:14:27.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.406666 12584 fs_manager.cc:362] Metadata directory not provided
16:14:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.406908 12584 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:27.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.407436 12584 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:27.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.411497 12584 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/instance:
16:14:27.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3567a5ad7a164c1cba57b85c36f6f434"
16:14:27.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:27 on dist-test-slave-4vz6"
16:14:27.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.413059 12584 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal/instance:
16:14:27.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3567a5ad7a164c1cba57b85c36f6f434"
16:14:27.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:27 on dist-test-slave-4vz6"
16:14:27.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.416631 12575 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } attempt: 7
16:14:27.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.421518 12584 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.002s
16:14:27.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.423633 12575 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:27.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.427464 12608 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:27.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.428678 12584 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.001s
16:14:27.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.429011 12584 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:27.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3567a5ad7a164c1cba57b85c36f6f434"
16:14:27.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:27 on dist-test-slave-4vz6"
16:14:27.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.429363 12584 fs_report.cc:389] FS layout report
16:14:27.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/data
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:27.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.502002 12584 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:27.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.507942 12584 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:27.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.508464 12584 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:27.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.582113 12584 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.125:38177
16:14:27.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.582183 12659 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.125:38177 every 8 connection(s)
16:14:27.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.585233 12584 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/info.pb
16:14:27.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.591470 12660 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:14:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.595722 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 12584
16:14:27.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.596191 12477 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal/instance
16:14:27.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.602391 12660 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } has no permanent_uuid. Determining permanent_uuid...
16:14:27.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.47.124@KRBTEST.COM; defaulting to no policy
16:14:27.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:27.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.47.124@KRBTEST.COM" created.
16:14:27.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:27 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104867, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.125@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:27.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.638103 12660 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } has no permanent_uuid. Determining permanent_uuid...
16:14:27.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:27.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:27.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:27.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:27 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104867, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.125@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:27.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.47.124@KRBTEST.COM; defaulting to no policy
16:14:27.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:27.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.47.124@KRBTEST.COM" created.
16:14:27.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.666930 12660 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } has no permanent_uuid. Determining permanent_uuid...
16:14:27.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.669334 12612 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:27.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.673689 12660 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.124:46043: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:27.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.683420 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data
16:14:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:27.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/logs
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/info.pb
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.124:46043
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.124
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.124
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:27.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:14:27.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:27.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.702185 11900 debug-util.cc:398] Leaking SignalData structure 0x7b08000afae0 after lost signal to thread 11835
16:14:27.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.702811 11900 debug-util.cc:398] Leaking SignalData structure 0x7b08000a40e0 after lost signal to thread 11903
16:14:27.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.723209 12660 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } attempt: 1
16:14:27.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.728478 12660 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.124:46043: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:27.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.796897 12660 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } attempt: 2
16:14:27.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.802909 12660 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.124:46043: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:27.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:27.894326 12660 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } attempt: 3
16:14:27.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:27.900000 12660 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.124:46043: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:28.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:28.044051 12660 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } attempt: 4
16:14:28.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.049453 12660 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.124:46043: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:28.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.064333 12670 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:28.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.064865 12670 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:28.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.065883 12670 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:28.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.101423 12670 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:14:28.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.101660 12670 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:28.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.101944 12670 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:28.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.102155 12670 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:28.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.102365 12670 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:28.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.102564 12670 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:28.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.102725 12670 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42857/jwks.json
16:14:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:28.144930 12670 master_runner.cc:387] Master server non-default flags:
16:14:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/logs
16:14:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data
16:14:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:28.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:14:28.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:28.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:28.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:28.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.124
16:14:28.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:28.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:28.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:28.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.124:46043
16:14:28.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:28.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/info.pb
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.124
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:28.146390 12670 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:28.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:28.147444 12670 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:28.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.162058 12678 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:28.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:28.348906 12660 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } attempt: 5
16:14:28.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.355731 12660 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.124:46043: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:28.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:28.903311 12660 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } attempt: 6
16:14:28.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.913079 12660 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.124:46043: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:28.166649 12670 server_base.cc:1061] running on GCE node
16:14:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:28.164772 12679 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:29.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:29.443693 12681 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:29.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.447830 12670 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:29.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.451159 12670 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:29.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.452606 12670 hybrid_clock.cc:648] HybridClock initialized: now 1795104869452547 us; error 39 us; skew 500 ppm
16:14:29.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.454052 12670 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:29.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:29 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104869, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:29.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.462196 12670 init.cc:377] Logged in from keytab as kudu/127.12.47.124@KRBTEST.COM (short username kudu)
16:14:29.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.468487 12670 webserver.cc:577] Webserver started at http://127.12.47.124:33711/ using document root <none> and password file <none>
16:14:29.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.469758 12670 fs_manager.cc:362] Metadata directory not provided
16:14:29.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.470019 12670 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:29.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.470595 12670 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:29.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.477372 12670 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/instance:
16:14:29.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5f92d38aabf4855966756a73fad5f87"
16:14:29.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:29 on dist-test-slave-4vz6"
16:14:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.478977 12670 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal/instance:
16:14:29.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5f92d38aabf4855966756a73fad5f87"
16:14:29.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:29 on dist-test-slave-4vz6"
16:14:29.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.490891 12670 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.009s sys 0.000s
16:14:29.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.499651 12693 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:29.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.501148 12670 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.000s
16:14:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.501579 12670 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5f92d38aabf4855966756a73fad5f87"
16:14:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:29 on dist-test-slave-4vz6"
16:14:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.502031 12670 fs_report.cc:389] FS layout report
16:14:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/data
16:14:29.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:29.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:29.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.514037 12575 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } attempt: 8
16:14:29.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:29 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104864, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.126@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:29.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.552172 12575 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } has no permanent_uuid. Determining permanent_uuid...
16:14:29.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:29.561236 12575 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.124:46043: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:29.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.587132 12670 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:29.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.592929 12670 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:29.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.593796 12670 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:29.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.626840 12575 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } attempt: 1
16:14:29.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:29.634927 12575 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.124:46043: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:29.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.688402 12575 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } attempt: 2
16:14:29.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:29.697419 12575 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.12.47.124:46043: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.702330 12670 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.124:46043
16:14:29.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.702392 12747 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.124:46043 every 8 connection(s)
16:14:29.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.705855 12670 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/info.pb
16:14:29.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.712729 12748 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:14:29.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.713209 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 12670
16:14:29.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.713675 12477 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal/instance
16:14:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.730743 12748 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } has no permanent_uuid. Determining permanent_uuid...
16:14:29.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:29 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104869, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.124@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:29.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:29 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104862, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:29.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.773373 12748 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } has no permanent_uuid. Determining permanent_uuid...
16:14:29.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.774209 12575 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } attempt: 3
16:14:29.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:29 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104862, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:29.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:29 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104869, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.124@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:29.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:29 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104864, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.126@KRBTEST.COM for kudu/127.12.47.124@KRBTEST.COM
16:14:29.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.803025 12748 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } has no permanent_uuid. Determining permanent_uuid...
16:14:29.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.812760 12575 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Bootstrap starting.
16:14:29.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.822530 12575 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Neither blocks nor log segments found. Creating new log.
16:14:29.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:29 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104862, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.47.124@KRBTEST.COM
16:14:29.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:29 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104869, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.124@KRBTEST.COM for kudu/127.12.47.124@KRBTEST.COM
16:14:29.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.825132 12575 log.cc:826] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Log is configured to *not* fsync() on all Append() calls
16:14:29.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.838156 12575 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: No bootstrap required, opened a new log
16:14:29.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.847725 12748 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Bootstrap starting.
16:14:29.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.855720 12748 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Neither blocks nor log segments found. Creating new log.
16:14:29.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.858155 12748 log.cc:826] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Log is configured to *not* fsync() on all Append() calls
16:14:29.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.864203 12748 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: No bootstrap required, opened a new log
16:14:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.867306 12575 raft_consensus.cc:360] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.868142 12575 raft_consensus.cc:386] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:14:29.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.868331 12575 raft_consensus.cc:748] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 127f4583c0704b96b6ee60f15dbd6a5f, State: Initialized, Role: FOLLOWER
16:14:29.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.869086 12575 consensus_queue.cc:261] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:29.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.872457 12760 sys_catalog.cc:455] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.873783 12760 sys_catalog.cc:458] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: This master's current role is: FOLLOWER
16:14:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.874269 12575 sys_catalog.cc:565] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: configured and running, proceeding with master startup.
16:14:29.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.890130 12748 raft_consensus.cc:360] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:29.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.890812 12748 raft_consensus.cc:386] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:14:29.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.891227 12748 raft_consensus.cc:748] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5f92d38aabf4855966756a73fad5f87, State: Initialized, Role: FOLLOWER
16:14:29.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.892153 12748 consensus_queue.cc:261] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:29.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.895641 12769 sys_catalog.cc:455] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:29.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.899201 12769 sys_catalog.cc:458] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: This master's current role is: FOLLOWER
16:14:29.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.897848 12748 sys_catalog.cc:565] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: configured and running, proceeding with master startup.
16:14:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:29.916551 12772 catalog_manager.cc:1609] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:14:29.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:29.917212 12772 catalog_manager.cc:898] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:14:29.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.918628 12760 raft_consensus.cc:499] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:14:29.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.919128 12760 raft_consensus.cc:521] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:29.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.921700 12760 leader_election.cc:290] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177), c5f92d38aabf4855966756a73fad5f87 (127.12.47.124:46043)
16:14:29.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.923126 12723 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c5f92d38aabf4855966756a73fad5f87" is_pre_election: true
16:14:29.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.924022 12723 raft_consensus.cc:2490] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 127f4583c0704b96b6ee60f15dbd6a5f in term 0.
16:14:29.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.925815 12527 leader_election.cc:304] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 127f4583c0704b96b6ee60f15dbd6a5f, c5f92d38aabf4855966756a73fad5f87; no voters:
16:14:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.926926 12760 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 0 FOLLOWER]: Leader pre-election won for term 1
16:14:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.927285 12760 raft_consensus.cc:499] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:14:29.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.927613 12760 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 0 FOLLOWER]: Advancing to term 1
16:14:29.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.936677 12760 raft_consensus.cc:521] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:29.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.941670 12723 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c5f92d38aabf4855966756a73fad5f87"
16:14:29.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.942237 12723 raft_consensus.cc:3096] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 0 FOLLOWER]: Advancing to term 1
16:14:29.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:29.956581 12529 tablet.cc:2554] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:14:29.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.960886 12760 leader_election.cc:290] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 1 election: Requested vote from peers 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177), c5f92d38aabf4855966756a73fad5f87 (127.12.47.124:46043)
16:14:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.970924 12723 raft_consensus.cc:2490] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 127f4583c0704b96b6ee60f15dbd6a5f in term 1.
16:14:29.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.974102 12660 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } attempt: 7
16:14:29.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.974617 12527 leader_election.cc:304] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 127f4583c0704b96b6ee60f15dbd6a5f, c5f92d38aabf4855966756a73fad5f87; no voters:
16:14:29.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.976223 12760 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Leader election won for term 1
16:14:29.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.994243 12760 raft_consensus.cc:705] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 LEADER]: Becoming Leader. State: Replica: 127f4583c0704b96b6ee60f15dbd6a5f, State: Running, Role: LEADER
16:14:29.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.995666 12760 consensus_queue.cc:238] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:29.914904 12506 master_runner.cc:187] Error getting master registration for 127.12.47.125:38177: OK, instance_id { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" instance_seqno: 1795104867562900 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:14:30.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.008807 12506 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:14:30.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.020059 12780 sys_catalog.cc:455] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 127f4583c0704b96b6ee60f15dbd6a5f. Latest consensus state: current_term: 1 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:30.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.024258 12780 sys_catalog.cc:458] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: This master's current role is: LEADER
16:14:30.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.041390 12790 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:14:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.044703 12787 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
16:14:30.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.054013 12787 catalog_manager.cc:1535] Initializing Kudu cluster ID...
16:14:30.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:30 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104867, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.125@KRBTEST.COM for kudu/127.12.47.124@KRBTEST.COM
16:14:30.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.078291 12760 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [LEADER]: Connected to new peer: Peer: permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:14:30.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:30.102818 12702 tablet.cc:2554] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:14:30.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.109062 12769 sys_catalog.cc:455] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 127f4583c0704b96b6ee60f15dbd6a5f. Latest consensus state: current_term: 1 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:30.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.109833 12769 sys_catalog.cc:458] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: This master's current role is: FOLLOWER
16:14:30.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.124961 12760 sys_catalog.cc:455] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:30.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.125706 12760 sys_catalog.cc:458] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: This master's current role is: LEADER
16:14:30.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.151523 12769 sys_catalog.cc:455] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:30.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.152182 12769 sys_catalog.cc:458] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: This master's current role is: FOLLOWER
16:14:30.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.152828 12660 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Bootstrap starting.
16:14:30.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.172130 12660 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Neither blocks nor log segments found. Creating new log.
16:14:30.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.174496 12660 log.cc:826] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Log is configured to *not* fsync() on all Append() calls
16:14:30.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.067795 12670 master_runner.cc:187] Error getting master registration for 127.12.47.125:38177: OK, instance_id { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" instance_seqno: 1795104867562900 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:14:30.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.182906 12670 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:14:30.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.185844 12660 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: No bootstrap required, opened a new log
16:14:30.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.225728 12787 catalog_manager.cc:1398] Generated new cluster ID: 2dd45602808940128709b885b85b6703
16:14:30.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.226011 12787 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
16:14:30.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.253186 12660 raft_consensus.cc:360] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:30.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.254123 12660 raft_consensus.cc:386] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:14:30.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.254429 12660 raft_consensus.cc:748] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3567a5ad7a164c1cba57b85c36f6f434, State: Initialized, Role: FOLLOWER
16:14:30.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.255474 12660 consensus_queue.cc:261] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:30.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.262518 12660 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: configured and running, proceeding with master startup.
16:14:30.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.262869 12787 catalog_manager.cc:1421] Generated new certificate authority record
16:14:30.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.263029 12632 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [term 0 FOLLOWER]: Advancing to term 1
16:14:30.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.264856 12787 catalog_manager.cc:1555] Loading token signing keys...
16:14:30.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.265578 12634 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3567a5ad7a164c1cba57b85c36f6f434"
16:14:30.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.267740 12635 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3567a5ad7a164c1cba57b85c36f6f434" is_pre_election: true
16:14:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:30.275278 12614 tablet.cc:2554] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:14:30.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.261257 12797 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:30.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.282102 12797 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: This master's current role is: FOLLOWER
16:14:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.283672 12789 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
16:14:30.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.341391 12797 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 127f4583c0704b96b6ee60f15dbd6a5f. Latest consensus state: current_term: 1 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:30.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.344136 12797 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: This master's current role is: FOLLOWER
16:14:30.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.345525 12804 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:30.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.346776 12804 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: This master's current role is: FOLLOWER
16:14:30.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.362540 12789 sys_catalog.cc:455] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:30.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.363220 12789 sys_catalog.cc:458] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: This master's current role is: LEADER
16:14:30.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.382809 12787 catalog_manager.cc:6237] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Generated new TSK 0
16:14:30.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.384579 12787 catalog_manager.cc:1565] Initializing in-progress tserver states...
16:14:30.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.390204 12812 catalog_manager.cc:1310] Loaded cluster ID: 2dd45602808940128709b885b85b6703
16:14:30.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.390442 12812 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: loading cluster ID for follower catalog manager: success
16:14:30.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.400527 12812 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: acquiring CA information for follower catalog manager: success
16:14:30.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.405210 12812 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:14:30.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.47.65@KRBTEST.COM; defaulting to no policy
16:14:30.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:30.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.47.65@KRBTEST.COM" created.
16:14:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:30.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.47.65@KRBTEST.COM; defaulting to no policy
16:14:30.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:30.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.47.65@KRBTEST.COM" created.
16:14:30.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:30.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:30.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:30.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:30.538784 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/wal
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/data
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/logs
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/data/info.pb
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.65:0
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.47.65
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.65
16:14:30.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.65
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:30.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:31.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:31.062119 12790 catalog_manager.cc:1310] Loaded cluster ID: 2dd45602808940128709b885b85b6703
16:14:31.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:31.062711 12790 catalog_manager.cc:1603] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: loading cluster ID for follower catalog manager: success
16:14:31.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:31.070598 12790 catalog_manager.cc:1625] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: acquiring CA information for follower catalog manager: success
16:14:31.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:31.076691 12819 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:31.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:31.077666 12819 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:31.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:31.078896 12819 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:31.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:31.086851 12790 catalog_manager.cc:1653] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:14:31.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:31.141728 12819 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:31.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:31.143739 12819 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.47.65
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:31.218685 12819 tablet_server_runner.cc:78] Tablet server non-default flags:
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/logs
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/data
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/wal
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.65
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:31.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.65:0
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/data/info.pb
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.65
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:31.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.47.65
16:14:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:14:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:31.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:31.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:31.224589 12819 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:31.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:31.227190 12819 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:31.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:31.290964 12827 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:34.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:34 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104874, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:34.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:34.855440 12519 init.cc:295] Successfully reacquired a new kerberos TGT
16:14:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:32.673147 12825 debug-util.cc:398] Leaking SignalData structure 0x7b0800035ce0 after lost signal to thread 12819
16:14:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:35.202641 12825 kernel_stack_watchdog.cc:198] Thread 12819 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 370ms:
16:14:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:14:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:14:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:14:35.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:14:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:31.293057 12826 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:35.205467 12819 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 3.905s user 0.002s sys 0.000s
16:14:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:35.205910 12819 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 3.905s user 0.002s sys 0.001s
16:14:35.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:35.216598 12830 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:35.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:35.220952 12828 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3920 milliseconds
16:14:35.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.220988 12819 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:35.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.222479 12819 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:35.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.225462 12819 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:35.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.226851 12819 hybrid_clock.cc:648] HybridClock initialized: now 1795104875226806 us; error 50 us; skew 500 ppm
16:14:35.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.228036 12819 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:35.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:35 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104875, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:35.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.234771 12819 init.cc:377] Logged in from keytab as kudu/127.12.47.65@KRBTEST.COM (short username kudu)
16:14:35.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.241333 12819 webserver.cc:577] Webserver started at http://127.12.47.65:37703/ using document root <none> and password file <none>
16:14:35.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.242537 12819 fs_manager.cc:362] Metadata directory not provided
16:14:35.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.242820 12819 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:35.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.243221 12819 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:35.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.247717 12819 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/data/instance:
16:14:35.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c0bdd2da57349af93defb40ca49f821"
16:14:35.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:35 on dist-test-slave-4vz6"
16:14:35.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.248891 12819 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/wal/instance:
16:14:35.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c0bdd2da57349af93defb40ca49f821"
16:14:35.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:35 on dist-test-slave-4vz6"
16:14:35.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.258062 12819 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.004s sys 0.003s
16:14:35.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.264132 12843 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:35.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.265908 12819 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.001s
16:14:35.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.266445 12819 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/wal
16:14:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1c0bdd2da57349af93defb40ca49f821"
16:14:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:35 on dist-test-slave-4vz6"
16:14:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.266956 12819 fs_report.cc:389] FS layout report
16:14:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:35.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/wal
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/wal
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/data/data
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:35.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:35.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.327211 12819 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:35.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.337563 12819 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:35.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.338105 12819 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:35.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.341032 12819 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:14:35.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.346035 12819 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:14:35.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.346270 12819 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:14:35.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.346625 12819 ts_tablet_manager.cc:662] Registered 0 tablets
16:14:35.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.347009 12819 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:14:35.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.521968 12819 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.65:37487
16:14:35.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.522267 12956 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.65:37487 every 8 connection(s)
16:14:35.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.525599 12819 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/data/info.pb
16:14:35.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.528056 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 12819
16:14:35.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.528597 12477 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-0/wal/instance
16:14:35.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.47.66@KRBTEST.COM; defaulting to no policy
16:14:35.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:35 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104875, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.65@KRBTEST.COM for kudu/127.12.47.124@KRBTEST.COM
16:14:35.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:35.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.47.66@KRBTEST.COM" created.
16:14:35.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:35 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104875, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.65@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:35 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104875, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.65@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:35.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.608816 12957 heartbeater.cc:345] Connected to a master server at 127.12.47.124:46043
16:14:35.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.609288 12957 heartbeater.cc:462] Registering TS with master...
16:14:35.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.610700 12957 heartbeater.cc:508] Master 127.12.47.124:46043 requested a full tablet report, sending...
16:14:35.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.615590 12713 ts_manager.cc:210] Registered new tserver with Master: 1c0bdd2da57349af93defb40ca49f821 (127.12.47.65:37487)
16:14:35.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.621160 12959 heartbeater.cc:345] Connected to a master server at 127.12.47.126:33337
16:14:35.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.621531 12959 heartbeater.cc:462] Registering TS with master...
16:14:35.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.622512 12959 heartbeater.cc:508] Master 127.12.47.126:33337 requested a full tablet report, sending...
16:14:35.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.623160 12958 heartbeater.cc:345] Connected to a master server at 127.12.47.125:38177
16:14:35.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.623561 12958 heartbeater.cc:462] Registering TS with master...
16:14:35.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.624493 12958 heartbeater.cc:508] Master 127.12.47.125:38177 requested a full tablet report, sending...
16:14:35.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.625384 12540 ts_manager.cc:210] Registered new tserver with Master: 1c0bdd2da57349af93defb40ca49f821 (127.12.47.65:37487)
16:14:35.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.628465 12540 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.47.65@KRBTEST.COM'} at 127.12.47.65:37643
16:14:35.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.629066 12625 ts_manager.cc:210] Registered new tserver with Master: 1c0bdd2da57349af93defb40ca49f821 (127.12.47.65:37487)
16:14:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:35.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:35.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.47.66@KRBTEST.COM; defaulting to no policy
16:14:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:35.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.47.66@KRBTEST.COM" created.
16:14:35.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:35.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:35.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:35.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:35.677484 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:35.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:35.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/wal
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/data
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/logs
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/data/info.pb
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.66:0
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.47.66
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.66
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.66
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:35.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:35.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:36.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:36.070693 12972 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:36.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:36.071360 12972 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:36.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:36.072299 12972 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:36.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:36.107971 12972 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:36.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:36.108913 12972 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.47.66
16:14:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:36.148808 12972 tablet_server_runner.cc:78] Tablet server non-default flags:
16:14:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/logs
16:14:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/data
16:14:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/wal
16:14:36.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.66
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.66:0
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/data/info.pb
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.66
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.47.66
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:36.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:36.150291 12972 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:36.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:36.151360 12972 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:36.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:36.164494 12980 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:36.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:36.632475 12959 heartbeater.cc:500] Master 127.12.47.126:33337 was elected leader, sending a full tablet report...
16:14:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:36.897845 11825 debug-util.cc:398] Leaking SignalData structure 0x7b0800081ae0 after lost signal to thread 11762
16:14:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:36.899435 11825 debug-util.cc:398] Leaking SignalData structure 0x7b080007dc60 after lost signal to thread 11828
16:14:37.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:37 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104877, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:37.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.412766 12602 init.cc:295] Successfully reacquired a new kerberos TGT
16:14:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:37.570681 12979 debug-util.cc:398] Leaking SignalData structure 0x7b0800035ce0 after lost signal to thread 12972
16:14:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:37.857813 12979 kernel_stack_watchdog.cc:198] Thread 12972 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 401ms:
16:14:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:14:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:14:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:14:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:14:37.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:36.164505 12981 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:37.859290 12972 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.694s user 0.001s sys 0.000s
16:14:37.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:37.859653 12972 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.694s user 0.001s sys 0.000s
16:14:37.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:37.873193 12982 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1708 milliseconds
16:14:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:37.873980 12984 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:37.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.873934 12972 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:37.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.876055 12972 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:37.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.878871 12972 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:37.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.880478 12972 hybrid_clock.cc:648] HybridClock initialized: now 1795104877880401 us; error 77 us; skew 500 ppm
16:14:37.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.882023 12972 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:37.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:37 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104877, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:37.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.888576 12972 init.cc:377] Logged in from keytab as kudu/127.12.47.66@KRBTEST.COM (short username kudu)
16:14:37.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.896695 12972 webserver.cc:577] Webserver started at http://127.12.47.66:41447/ using document root <none> and password file <none>
16:14:37.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.898172 12972 fs_manager.cc:362] Metadata directory not provided
16:14:37.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.898610 12972 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:37.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.899322 12972 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:37.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.906476 12972 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/data/instance:
16:14:37.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acc39da2179b470e918cf0f37a34954c"
16:14:37.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:37 on dist-test-slave-4vz6"
16:14:37.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.908141 12972 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/wal/instance:
16:14:37.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acc39da2179b470e918cf0f37a34954c"
16:14:37.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:37 on dist-test-slave-4vz6"
16:14:37.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.918349 12972 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.011s sys 0.001s
16:14:37.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.926024 12993 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:37.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.927585 12972 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
16:14:37.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.927990 12972 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/wal
16:14:37.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "acc39da2179b470e918cf0f37a34954c"
16:14:37.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:37 on dist-test-slave-4vz6"
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:37.928437 12972 fs_report.cc:389] FS layout report
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/wal
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/wal
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/data/data
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:37.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:38.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.002679 12972 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:38.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.008327 12972 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:38.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.008729 12972 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:38.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.012310 12972 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:14:38.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.017547 12972 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:14:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.017818 12972 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:14:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.018141 12972 ts_tablet_manager.cc:662] Registered 0 tablets
16:14:38.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.018349 12972 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:14:38.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.222878 12972 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.66:39193
16:14:38.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.224560 13106 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.66:39193 every 8 connection(s)
16:14:38.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.226893 12972 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/data/info.pb
16:14:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.232570 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 12972
16:14:38.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.233032 12477 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-1/wal/instance
16:14:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:38 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104877, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.66@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:38.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.47.67@KRBTEST.COM; defaulting to no policy
16:14:38.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:38 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104877, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.66@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:38.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:38.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.47.67@KRBTEST.COM" created.
16:14:38.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:38 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104877, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.66@KRBTEST.COM for kudu/127.12.47.124@KRBTEST.COM
16:14:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.325984 13108 heartbeater.cc:345] Connected to a master server at 127.12.47.125:38177
16:14:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.326489 13108 heartbeater.cc:462] Registering TS with master...
16:14:38.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.327844 13108 heartbeater.cc:508] Master 127.12.47.125:38177 requested a full tablet report, sending...
16:14:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.331660 12625 ts_manager.cc:210] Registered new tserver with Master: acc39da2179b470e918cf0f37a34954c (127.12.47.66:39193)
16:14:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.331992 13111 heartbeater.cc:345] Connected to a master server at 127.12.47.126:33337
16:14:38.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.332309 13111 heartbeater.cc:462] Registering TS with master...
16:14:38.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.333256 13111 heartbeater.cc:508] Master 127.12.47.126:33337 requested a full tablet report, sending...
16:14:38.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.335489 12540 ts_manager.cc:210] Registered new tserver with Master: acc39da2179b470e918cf0f37a34954c (127.12.47.66:39193)
16:14:38.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.336772 12540 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.47.66@KRBTEST.COM'} at 127.12.47.66:50369
16:14:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.340963 13107 heartbeater.cc:345] Connected to a master server at 127.12.47.124:46043
16:14:38.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.341389 13107 heartbeater.cc:462] Registering TS with master...
16:14:38.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.342093 13107 heartbeater.cc:508] Master 127.12.47.124:46043 requested a full tablet report, sending...
16:14:38.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.345129 12713 ts_manager.cc:210] Registered new tserver with Master: acc39da2179b470e918cf0f37a34954c (127.12.47.66:39193)
16:14:38.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:38.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:38.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.47.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:38.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.47.67@KRBTEST.COM; defaulting to no policy
16:14:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:38.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.47.67@KRBTEST.COM" created.
16:14:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:14:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:38.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.47.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab.
16:14:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.395280 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:38.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/wal
16:14:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/data
16:14:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/logs
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/data/info.pb
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.67:0
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.47.67
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.67
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:38.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.67
16:14:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:38.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:38.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:38.836910 13121 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:38.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:38.837488 13121 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:38.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:38.838214 13121 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:38.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:38.874751 13121 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:38.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:38.876065 13121 flags.cc:432] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.47.67
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.919988 13121 tablet_server_runner.cc:78] Tablet server non-default flags:
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/logs
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/data
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/wal
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:38.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.67
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.67:0
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/data/info.pb
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.67
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:14:38.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.47.67
16:14:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:14:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:38.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:38.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:38.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:38.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:38.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.921203 13121 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:38.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:38.922590 13121 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:38.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:38.941250 13130 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:39.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:39.341831 13111 heartbeater.cc:500] Master 127.12.47.126:33337 was elected leader, sending a full tablet report...
16:14:39.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:39 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104879, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:39.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:39.468741 12689 init.cc:295] Successfully reacquired a new kerberos TGT
16:14:39.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:39.761667 12179 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae920 after lost signal to thread 12053
16:14:39.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:39.763100 12179 debug-util.cc:398] Leaking SignalData structure 0x7b08000baaa0 after lost signal to thread 12182
16:14:40.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:40.491124 11980 debug-util.cc:398] Leaking SignalData structure 0x7b080007d340 after lost signal to thread 11911
16:14:40.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:40.492571 11980 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7400 after lost signal to thread 11983
16:14:41.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:38.943183 13129 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:41.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:40.348439 13128 debug-util.cc:398] Leaking SignalData structure 0x7b0800035ce0 after lost signal to thread 13121
16:14:41.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:41.202150 13131 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2257 milliseconds
16:14:41.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:41.202404 13121 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 2.260s user 0.690s sys 1.176s
16:14:41.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:41.204377 13121 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 2.262s user 0.690s sys 1.176s
16:14:41.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:41.204461 13132 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:41.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.204715 13121 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:41.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.209939 13121 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:41.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.234341 13121 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:41.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.235803 13121 hybrid_clock.cc:648] HybridClock initialized: now 1795104881235759 us; error 55 us; skew 500 ppm
16:14:41.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.237008 13121 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:41.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:41 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104881, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:41.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.255982 13121 init.cc:377] Logged in from keytab as kudu/127.12.47.67@KRBTEST.COM (short username kudu)
16:14:41.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.282337 13121 webserver.cc:577] Webserver started at http://127.12.47.67:44149/ using document root <none> and password file <none>
16:14:41.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.283733 13121 fs_manager.cc:362] Metadata directory not provided
16:14:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.284008 13121 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:41.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.284601 13121 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
16:14:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.291659 13121 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/data/instance:
16:14:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac9f2cf9bc7543ad949581708041b41f"
16:14:41.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:41 on dist-test-slave-4vz6"
16:14:41.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.293422 13121 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/wal/instance:
16:14:41.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac9f2cf9bc7543ad949581708041b41f"
16:14:41.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:41 on dist-test-slave-4vz6"
16:14:41.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.311736 13121 fs_manager.cc:696] Time spent creating directory manager: real 0.017s user 0.009s sys 0.004s
16:14:41.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.329902 13145 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:41.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.331406 13121 fs_manager.cc:730] Time spent opening block manager: real 0.015s user 0.005s sys 0.005s
16:14:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.331851 13121 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/wal
16:14:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ac9f2cf9bc7543ad949581708041b41f"
16:14:41.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:41 on dist-test-slave-4vz6"
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.332316 13121 fs_report.cc:389] FS layout report
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/wal
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/wal
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/data/data
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:41.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:41.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.448774 13121 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:41.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.465027 13121 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:41.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.465657 13121 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:41.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.476130 13121 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:14:41.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.494828 13121 ts_tablet_manager.cc:631] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:14:41.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.495162 13121 ts_tablet_manager.cc:577] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.000s
16:14:41.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.495607 13121 ts_tablet_manager.cc:662] Registered 0 tablets
16:14:41.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.495915 13121 ts_tablet_manager.cc:641] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:14:41.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.908262 13121 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.67:43031
16:14:41.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.908387 13257 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.67:43031 every 8 connection(s)
16:14:41.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.912158 13121 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/data/info.pb
16:14:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.915438 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 13121
16:14:41.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:41.916260 12477 external_mini_cluster.cc:1448] Reading /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/ts-2/wal/instance
16:14:42.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:42 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104881, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.67@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:42.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:42 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104881, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.67@KRBTEST.COM for kudu/127.12.47.124@KRBTEST.COM
16:14:42.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.047672 13259 heartbeater.cc:345] Connected to a master server at 127.12.47.125:38177
16:14:42.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.048143 13259 heartbeater.cc:462] Registering TS with master...
16:14:42.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.050686 13259 heartbeater.cc:508] Master 127.12.47.125:38177 requested a full tablet report, sending...
16:14:42.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.053893 12625 ts_manager.cc:210] Registered new tserver with Master: ac9f2cf9bc7543ad949581708041b41f (127.12.47.67:43031)
16:14:42.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:42 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104881, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.67@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.079347 13264 heartbeater.cc:345] Connected to a master server at 127.12.47.126:33337
16:14:42.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.079720 13264 heartbeater.cc:462] Registering TS with master...
16:14:42.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.081524 13264 heartbeater.cc:508] Master 127.12.47.126:33337 requested a full tablet report, sending...
16:14:42.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.084154 12540 ts_manager.cc:210] Registered new tserver with Master: ac9f2cf9bc7543ad949581708041b41f (127.12.47.67:43031)
16:14:42.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.085805 12540 master_service.cc:507] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.47.67@KRBTEST.COM'} at 127.12.47.67:40579
16:14:42.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.105152 13258 heartbeater.cc:345] Connected to a master server at 127.12.47.124:46043
16:14:42.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.105522 13258 heartbeater.cc:462] Registering TS with master...
16:14:42.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.106148 13258 heartbeater.cc:508] Master 127.12.47.124:46043 requested a full tablet report, sending...
16:14:42.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.108793 12713 ts_manager.cc:210] Registered new tserver with Master: ac9f2cf9bc7543ad949581708041b41f (127.12.47.67:43031)
16:14:42.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.114789 12477 external_mini_cluster.cc:955] 3 TS(s) registered with all masters
16:14:42.143 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:42.144 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
16:14:42.155 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
16:14:42.156 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
16:14:42.159 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.12.47.126"
port: 33337
}
bound_http_address {
host: "127.12.47.126"
port: 37859
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.12.47.125"
port: 38177
}
bound_http_address {
host: "127.12.47.125"
port: 36133
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.12.47.124"
port: 46043
}
bound_http_address {
host: "127.12.47.124"
port: 33711
}
}
}
16:14:42.159 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:14:42.161 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.12.47.65"
port: 37487
}
bound_http_address {
host: "127.12.47.65"
port: 37703
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.12.47.66"
port: 39193
}
bound_http_address {
host: "127.12.47.66"
port: 41447
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.12.47.67"
port: 43031
}
bound_http_address {
host: "127.12.47.67"
port: 44149
}
}
}
16:14:42.161 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:14:42.162 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:14:42.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:42 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104882, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:42.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:14:42.205 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:42.207 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc
16:14:42.219 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:14:42.220 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc
16:14:42.221 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:14:42.255 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
16:14:42.260 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:14:42.261 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:14:42.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:42 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104882, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:42.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:14:42.312 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:42.312 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc
16:14:42.314 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:14:42.314 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.47.126:33337
16:14:42.316 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:14:42.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.317111 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 12506
16:14:42.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.369151 12997 connection.cc:573] client connection to 127.12.47.126:33337 recv error: Network error: recv error from unknown peer: Connection reset by peer (error 104)
16:14:42.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.370456 13111 heartbeater.cc:648] Failed to heartbeat to 127.12.47.126:33337 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Connection reset by peer (error 104)
16:14:42.373 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:42.373 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.47.124:46043
16:14:42.374 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:14:42.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.374394 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 12670
16:14:42.411 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:42.411 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.47.125:38177
16:14:42.412 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:14:42.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.412547 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 12584
16:14:42.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:42.450 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.47.126:33337
16:14:42.456 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:14:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.458204 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:42.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/logs
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/info.pb
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.126:33337
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.126
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37859
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:42.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.126
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:14:42.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:42.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.676738 12958 heartbeater.cc:648] Failed to heartbeat to 127.12.47.125:38177 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:42.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.858182 13272 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:42.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.858896 13272 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:42.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.859689 13272 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:42.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.896464 13272 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:14:42.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.896800 13272 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:42.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.897266 13272 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:42.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.897704 13272 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:42.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.898065 13272 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:42.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.898427 13272 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:42.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.898684 13272 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42857/jwks.json
16:14:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.960588 13272 master_runner.cc:387] Master server non-default flags:
16:14:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/logs
16:14:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:42.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.126
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.126:33337
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:42.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/info.pb
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.126
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37859
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.962257 13272 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:42.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:42.963721 13272 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:42.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.976666 13281 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:43.079806 13259 heartbeater.cc:648] Failed to heartbeat to 127.12.47.125:38177 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:43.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:43.091737 13264 heartbeater.cc:500] Master 127.12.47.126:33337 was elected leader, sending a full tablet report...
16:14:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:42.980187 13282 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:44.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:44.378659 13280 debug-util.cc:398] Leaking SignalData structure 0x7b0800038620 after lost signal to thread 13272
16:14:44.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:44.656760 13280 kernel_stack_watchdog.cc:198] Thread 13272 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 395ms:
16:14:44.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:14:44.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:14:44.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:14:44.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:14:44.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:44.661933 13272 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.680s user 0.005s sys 0.003s
16:14:44.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:44.662371 13272 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.681s user 0.005s sys 0.003s
16:14:44.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:44.664785 13284 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:44.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.665181 13272 server_base.cc:1061] running on GCE node
16:14:44.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.666850 13272 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:44.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.669685 13272 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:44.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.671098 13272 hybrid_clock.cc:648] HybridClock initialized: now 1795104884671044 us; error 49 us; skew 500 ppm
16:14:44.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.671965 13272 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:44 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104884, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:44.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.678195 13272 init.cc:377] Logged in from keytab as kudu/127.12.47.126@KRBTEST.COM (short username kudu)
16:14:44.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.683519 13272 webserver.cc:577] Webserver started at http://127.12.47.126:37859/ using document root <none> and password file <none>
16:14:44.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.684420 13272 fs_manager.cc:362] Metadata directory not provided
16:14:44.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.684630 13272 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:44.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.696270 13272 fs_manager.cc:714] Time spent opening directory manager: real 0.009s user 0.003s sys 0.004s
16:14:44.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.702113 13300 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:44.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.703853 13272 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
16:14:44.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.704455 13272 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:44.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "127f4583c0704b96b6ee60f15dbd6a5f"
16:14:44.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:24 on dist-test-slave-4vz6"
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.706664 13272 fs_report.cc:389] FS layout report
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/data
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:44.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:44.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:44.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.765614 13272 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:44.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.771404 13272 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:44.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.771806 13272 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:44.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.845310 13272 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.126:33337
16:14:44.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.845463 13352 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.126:33337 every 8 connection(s)
16:14:44.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.848793 13272 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/info.pb
16:14:44.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.851845 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 13272
16:14:44.852 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:44.852 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.47.124:46043
16:14:44.853 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.854938 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/logs
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/info.pb
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:44.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:44.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:44.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:44.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.124:46043
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.124
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33711
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.124
16:14:44.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:14:44.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:44.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.866034 13353 sys_catalog.cc:263] Verifying existing consensus state
16:14:44.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.872305 13353 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Bootstrap starting.
16:14:44.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.912612 13353 log.cc:826] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Log is configured to *not* fsync() on all Append() calls
16:14:44.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.928699 13353 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:14:44.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.929613 13353 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Bootstrap complete.
16:14:44.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.954329 13353 raft_consensus.cc:360] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:44.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.956569 13353 raft_consensus.cc:748] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 127f4583c0704b96b6ee60f15dbd6a5f, State: Initialized, Role: FOLLOWER
16:14:44.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.957659 13353 consensus_queue.cc:261] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:44.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.963235 13357 sys_catalog.cc:455] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:44.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.964448 13357 sys_catalog.cc:458] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: This master's current role is: FOLLOWER
16:14:44.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:44.965256 13353 sys_catalog.cc:565] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: configured and running, proceeding with master startup.
16:14:45.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.005013 13368 catalog_manager.cc:1310] Loaded cluster ID: 2dd45602808940128709b885b85b6703
16:14:45.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.007961 13368 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: loading cluster ID for follower catalog manager: success
16:14:45.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:45 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104884, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.126@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.018043 13368 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: acquiring CA information for follower catalog manager: success
16:14:45.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.024139 13368 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:14:45.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.037030 13272 master_runner.cc:187] Error getting master registration for 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111),
16:14:45.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.066385 13272 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:14:45.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.156447 13264 heartbeater.cc:345] Connected to a master server at 127.12.47.126:33337
16:14:45.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.157071 13264 heartbeater.cc:500] Master 127.12.47.126:33337 was elected leader, sending a full tablet report...
16:14:45.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.163230 13318 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac9f2cf9bc7543ad949581708041b41f" instance_seqno: 1795104881857195) as {username='kudu', principal='kudu/127.12.47.67@KRBTEST.COM'} at 127.12.47.67:54419; Asking this server to re-register.
16:14:45.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.164846 13264 heartbeater.cc:462] Registering TS with master...
16:14:45.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.168282 13318 ts_manager.cc:210] Registered new tserver with Master: ac9f2cf9bc7543ad949581708041b41f (127.12.47.67:43031)
16:14:45.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:45 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104885, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:45.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.240186 12839 init.cc:295] Successfully reacquired a new kerberos TGT
16:14:45.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.245752 13355 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:45.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.246423 13355 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:45.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.247221 13355 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:45.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.287101 13355 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:14:45.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.287400 13355 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:45.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.287842 13355 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:45.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.288192 13355 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:45.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.288537 13355 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:45.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.288877 13355 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:45.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.289134 13355 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42857/jwks.json
16:14:45.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.330505 13355 master_runner.cc:387] Master server non-default flags:
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/logs
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.124
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.124:46043
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:45.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/info.pb
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.124
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33711
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.332177 13355 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:45.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.333566 13355 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:45.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.345435 13380 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:45.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.438081 13111 heartbeater.cc:345] Connected to a master server at 127.12.47.126:33337
16:14:45.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.441286 13318 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "acc39da2179b470e918cf0f37a34954c" instance_seqno: 1795104878181549) as {username='kudu', principal='kudu/127.12.47.66@KRBTEST.COM'} at 127.12.47.66:56833; Asking this server to re-register.
16:14:45.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.443113 13111 heartbeater.cc:462] Registering TS with master...
16:14:45.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.446679 13318 ts_manager.cc:210] Registered new tserver with Master: acc39da2179b470e918cf0f37a34954c (127.12.47.66:39193)
16:14:45.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.773610 12959 heartbeater.cc:345] Connected to a master server at 127.12.47.126:33337
16:14:45.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.777289 13318 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1c0bdd2da57349af93defb40ca49f821" instance_seqno: 1795104875481619) as {username='kudu', principal='kudu/127.12.47.65@KRBTEST.COM'} at 127.12.47.65:34415; Asking this server to re-register.
16:14:45.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.779381 12959 heartbeater.cc:462] Registering TS with master...
16:14:45.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:45.784483 13318 ts_manager.cc:210] Registered new tserver with Master: 1c0bdd2da57349af93defb40ca49f821 (127.12.47.65:37487)
16:14:46.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:46.696587 13396 raft_consensus.cc:499] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:14:46.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:46.700500 13396 raft_consensus.cc:521] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:46.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:46.760636 13302 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:46.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:46.764925 13396 leader_election.cc:290] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177), c5f92d38aabf4855966756a73fad5f87 (127.12.47.124:46043)
16:14:46.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:46.779644 13305 leader_election.cc:336] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c5f92d38aabf4855966756a73fad5f87 (127.12.47.124:46043): Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:46.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:46.781132 13302 leader_election.cc:336] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177): Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:46.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:46.782732 13302 leader_election.cc:304] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 127f4583c0704b96b6ee60f15dbd6a5f; no voters: 3567a5ad7a164c1cba57b85c36f6f434, c5f92d38aabf4855966756a73fad5f87
16:14:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:46.786099 13396 raft_consensus.cc:2771] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
16:14:47.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:46.748553 13379 debug-util.cc:398] Leaking SignalData structure 0x7b0800038620 after lost signal to thread 13355
16:14:47.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:47.319870 13355 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.973s user 0.535s sys 1.062s
16:14:47.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:47.320312 13355 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.974s user 0.535s sys 1.062s
16:14:47.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:45.347522 13381 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:47.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:47.324523 13382 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1978 milliseconds
16:14:47.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:47.325977 13383 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:47.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.326025 13355 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:47.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.327261 13355 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:47.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.329774 13355 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:47.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.331146 13355 hybrid_clock.cc:648] HybridClock initialized: now 1795104887331115 us; error 34 us; skew 500 ppm
16:14:47.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.332032 13355 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:47.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:47 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104887, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:47.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.338349 13355 init.cc:377] Logged in from keytab as kudu/127.12.47.124@KRBTEST.COM (short username kudu)
16:14:47.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.344326 13355 webserver.cc:577] Webserver started at http://127.12.47.124:33711/ using document root <none> and password file <none>
16:14:47.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.345404 13355 fs_manager.cc:362] Metadata directory not provided
16:14:47.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.345672 13355 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:47.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.354031 13355 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
16:14:47.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.358646 13408 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:47.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.359820 13355 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
16:14:47.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.360193 13355 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:47.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5f92d38aabf4855966756a73fad5f87"
16:14:47.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:29 on dist-test-slave-4vz6"
16:14:47.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.362196 13355 fs_report.cc:389] FS layout report
16:14:47.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:47.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/data
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:47.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:47.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.430229 13355 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:47.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.442875 13355 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:47.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.443528 13355 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:47.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.568799 13355 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.124:46043
16:14:47.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.568936 13461 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.124:46043 every 8 connection(s)
16:14:47.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.572858 13355 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/info.pb
16:14:47.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.582985 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 13355
16:14:47.584 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:47.584 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.47.125:38177
16:14:47.585 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:14:47.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.586720 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:47.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:47.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:47.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data
16:14:47.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:47.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:47.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:47.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:47.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:47.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/logs
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/info.pb
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.125:38177
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.125
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36133
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.125
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:47.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:14:47.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:47.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.596272 13462 sys_catalog.cc:263] Verifying existing consensus state
16:14:47.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.608086 13462 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Bootstrap starting.
16:14:47.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.649111 13462 log.cc:826] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Log is configured to *not* fsync() on all Append() calls
16:14:47.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.666555 13462 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:14:47.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.667701 13462 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Bootstrap complete.
16:14:47.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.693140 13462 raft_consensus.cc:360] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:47.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.696964 13462 raft_consensus.cc:748] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5f92d38aabf4855966756a73fad5f87, State: Initialized, Role: FOLLOWER
16:14:47.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.698273 13462 consensus_queue.cc:261] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:47.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.742267 13466 sys_catalog.cc:455] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:47.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.743096 13466 sys_catalog.cc:458] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: This master's current role is: FOLLOWER
16:14:47.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.743871 13462 sys_catalog.cc:565] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: configured and running, proceeding with master startup.
16:14:47.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:47 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104885, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.65@KRBTEST.COM for kudu/127.12.47.124@KRBTEST.COM
16:14:47.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.837718 12957 heartbeater.cc:345] Connected to a master server at 127.12.47.124:46043
16:14:47.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:47 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104887, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:47.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.894428 12989 init.cc:295] Successfully reacquired a new kerberos TGT
16:14:47.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:47 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104887, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.124@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:47.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.901870 13479 catalog_manager.cc:1310] Loaded cluster ID: 2dd45602808940128709b885b85b6703
16:14:47.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.902277 13479 catalog_manager.cc:1603] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: loading cluster ID for follower catalog manager: success
16:14:47.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.910347 13479 catalog_manager.cc:1625] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: acquiring CA information for follower catalog manager: success
16:14:47.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.917304 13479 catalog_manager.cc:1653] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:14:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.930509 13355 master_runner.cc:187] Error getting master registration for 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111),
16:14:47.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:47.961336 13355 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:14:48.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.122412 13464 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:48.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.123199 13464 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:48.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.124521 13464 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:48.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.177652 13258 heartbeater.cc:345] Connected to a master server at 127.12.47.124:46043
16:14:48.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.184165 13427 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac9f2cf9bc7543ad949581708041b41f" instance_seqno: 1795104881857195) as {username='kudu', principal='kudu/127.12.47.67@KRBTEST.COM'} at 127.12.47.67:60665; Asking this server to re-register.
16:14:48.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.184923 13464 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:14:48.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.185240 13464 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:48.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.185760 13464 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:48.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.186111 13464 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:48.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.186460 13464 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:48.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.186801 13464 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:48.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.187052 13464 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42857/jwks.json
16:14:48.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.189248 13258 heartbeater.cc:462] Registering TS with master...
16:14:48.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.192655 13426 ts_manager.cc:210] Registered new tserver with Master: ac9f2cf9bc7543ad949581708041b41f (127.12.47.67:43031)
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.231917 13464 master_runner.cc:387] Master server non-default flags:
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/logs
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.125
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:48.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.125:38177
16:14:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:48.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/info.pb
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.125
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36133
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:48.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.237803 13464 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.239352 13464 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:48.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.255527 13492 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.471457 13107 heartbeater.cc:345] Connected to a master server at 127.12.47.124:46043
16:14:48.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.475039 13426 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "acc39da2179b470e918cf0f37a34954c" instance_seqno: 1795104878181549) as {username='kudu', principal='kudu/127.12.47.66@KRBTEST.COM'} at 127.12.47.66:36079; Asking this server to re-register.
16:14:48.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.477510 13107 heartbeater.cc:462] Registering TS with master...
16:14:48.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.481752 13426 ts_manager.cc:210] Registered new tserver with Master: acc39da2179b470e918cf0f37a34954c (127.12.47.66:39193)
16:14:48.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.844200 13426 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1c0bdd2da57349af93defb40ca49f821" instance_seqno: 1795104875481619) as {username='kudu', principal='kudu/127.12.47.65@KRBTEST.COM'} at 127.12.47.65:55551; Asking this server to re-register.
16:14:48.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.846310 12957 heartbeater.cc:462] Registering TS with master...
16:14:48.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.849560 13426 ts_manager.cc:210] Registered new tserver with Master: 1c0bdd2da57349af93defb40ca49f821 (127.12.47.65:37487)
16:14:48.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.996229 13502 raft_consensus.cc:499] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:14:48.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:48.998091 13502 raft_consensus.cc:521] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.018761 13502 leader_election.cc:290] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177), c5f92d38aabf4855966756a73fad5f87 (127.12.47.124:46043)
16:14:49.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:49.035627 13302 leader_election.cc:336] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177): Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:49.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.048028 13437 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c5f92d38aabf4855966756a73fad5f87" is_pre_election: true
16:14:49.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.049278 13437 raft_consensus.cc:2490] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 127f4583c0704b96b6ee60f15dbd6a5f in term 1.
16:14:49.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.051882 13305 leader_election.cc:304] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 127f4583c0704b96b6ee60f15dbd6a5f, c5f92d38aabf4855966756a73fad5f87; no voters: 3567a5ad7a164c1cba57b85c36f6f434
16:14:49.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.053263 13502 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Leader pre-election won for term 2
16:14:49.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.053694 13502 raft_consensus.cc:499] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:14:49.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.054082 13502 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 1 FOLLOWER]: Advancing to term 2
16:14:49.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.067575 13502 raft_consensus.cc:521] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:49.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:49.075850 13302 leader_election.cc:336] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177): Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:49.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.078470 13437 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "c5f92d38aabf4855966756a73fad5f87"
16:14:49.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.079020 13437 raft_consensus.cc:3096] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 1 FOLLOWER]: Advancing to term 2
16:14:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.091564 13437 raft_consensus.cc:2490] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 127f4583c0704b96b6ee60f15dbd6a5f in term 2.
16:14:49.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.092325 13502 leader_election.cc:290] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 2 election: Requested vote from peers 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177), c5f92d38aabf4855966756a73fad5f87 (127.12.47.124:46043)
16:14:49.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.094276 13305 leader_election.cc:304] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 127f4583c0704b96b6ee60f15dbd6a5f, c5f92d38aabf4855966756a73fad5f87; no voters: 3567a5ad7a164c1cba57b85c36f6f434
16:14:49.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.096591 13502 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Leader election won for term 2
16:14:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.105304 13502 raft_consensus.cc:705] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 LEADER]: Becoming Leader. State: Replica: 127f4583c0704b96b6ee60f15dbd6a5f, State: Running, Role: LEADER
16:14:49.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.110124 13502 consensus_queue.cc:238] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:49.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.127461 13507 sys_catalog.cc:455] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 127f4583c0704b96b6ee60f15dbd6a5f. Latest consensus state: current_term: 2 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:49.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.128662 13507 sys_catalog.cc:458] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: This master's current role is: LEADER
16:14:49.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.139684 13437 raft_consensus.cc:1285] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 2 FOLLOWER]: Refusing update from remote peer 127f4583c0704b96b6ee60f15dbd6a5f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
16:14:49.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.142159 13507 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [LEADER]: Connected to new peer: Peer: permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
16:14:49.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.144404 13511 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
16:14:49.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:49.154142 13302 consensus_peers.cc:597] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f -> Peer 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177): Couldn't send request to peer 3567a5ad7a164c1cba57b85c36f6f434. Status: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
16:14:49.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.157915 13511 catalog_manager.cc:1535] Initializing Kudu cluster ID...
16:14:49.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.169497 13511 catalog_manager.cc:1310] Loaded cluster ID: 2dd45602808940128709b885b85b6703
16:14:49.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.171223 13511 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
16:14:49.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.175060 13513 sys_catalog.cc:455] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 127f4583c0704b96b6ee60f15dbd6a5f. Latest consensus state: current_term: 2 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:49.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.178263 13511 catalog_manager.cc:1555] Loading token signing keys...
16:14:49.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.186795 13513 sys_catalog.cc:458] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: This master's current role is: FOLLOWER
16:14:49.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.188496 13511 catalog_manager.cc:6248] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Loaded TSK: 0
16:14:49.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.190624 13502 sys_catalog.cc:455] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.191636 13502 sys_catalog.cc:458] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: This master's current role is: LEADER
16:14:49.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.192984 13511 catalog_manager.cc:1565] Initializing in-progress tserver states...
16:14:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.214184 13513 sys_catalog.cc:455] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:49.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:49.214958 13513 sys_catalog.cc:458] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: This master's current role is: FOLLOWER
16:14:50.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.071998 13272 master_runner.cc:187] Error getting master registration for 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111),
16:14:50.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.072365 13272 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:14:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:50.087631 13464 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 1.823s user 0.597s sys 0.970s
16:14:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:49.655592 13491 debug-util.cc:398] Leaking SignalData structure 0x7b0800038620 after lost signal to thread 13464
16:14:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:50.090167 13464 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 1.826s user 0.597s sys 0.970s
16:14:50.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.090428 13464 server_base.cc:1061] running on GCE node
16:14:50.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:50.091352 13495 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:50.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:48.270032 13493 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.095681 13464 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:50.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.098676 13464 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.100040 13464 hybrid_clock.cc:648] HybridClock initialized: now 1795104890100009 us; error 35 us; skew 500 ppm
16:14:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.100924 13464 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:50.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:50 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104890, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:50.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.107103 13464 init.cc:377] Logged in from keytab as kudu/127.12.47.125@KRBTEST.COM (short username kudu)
16:14:50.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.112740 13464 webserver.cc:577] Webserver started at http://127.12.47.125:36133/ using document root <none> and password file <none>
16:14:50.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.113758 13464 fs_manager.cc:362] Metadata directory not provided
16:14:50.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.114019 13464 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:50.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.122723 13464 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.001s sys 0.002s
16:14:50.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.127509 13527 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:50.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.128774 13464 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
16:14:50.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.129148 13464 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3567a5ad7a164c1cba57b85c36f6f434"
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:27 on dist-test-slave-4vz6"
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.131407 13464 fs_report.cc:389] FS layout report
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/data
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:50.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:50.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.190210 13464 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:50.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.196715 13464 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:50.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.197260 13464 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:50.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.299211 13464 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.125:38177
16:14:50.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.299256 13579 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.125:38177 every 8 connection(s)
16:14:50.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.302223 13464 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/info.pb
16:14:50.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.307778 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 13464
16:14:50.308 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.317400 13580 sys_catalog.cc:263] Verifying existing consensus state
16:14:50.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.325737 13580 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Bootstrap starting.
16:14:50.357 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.47.126:33337(127.12.47.126:33337)] connecting to peer
16:14:50.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.383409 13580 log.cc:826] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Log is configured to *not* fsync() on all Append() calls
16:14:50.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.409888 13580 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:14:50.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.410912 13580 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Bootstrap complete.
16:14:50.421 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.47.124:46043(127.12.47.124:46043)] connecting to peer
16:14:50.424 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.47.125:38177(127.12.47.125:38177)] connecting to peer
16:14:50.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.444782 13580 raft_consensus.cc:360] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:50.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.448117 13580 raft_consensus.cc:748] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3567a5ad7a164c1cba57b85c36f6f434, State: Initialized, Role: FOLLOWER
16:14:50.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.449532 13580 consensus_queue.cc:261] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:50.456 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.12.47.124:46043(127.12.47.124:46043)] Successfully connected to peer
16:14:50.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.464087 13111 heartbeater.cc:500] Master 127.12.47.126:33337 was elected leader, sending a full tablet report...
16:14:50.465 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.47.126:33337(127.12.47.126:33337)] Successfully connected to peer
16:14:50.472 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.12.47.125:38177(127.12.47.125:38177)] Successfully connected to peer
16:14:50.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.563205 13587 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:50.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.565236 13587 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: This master's current role is: FOLLOWER
16:14:50.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.586623 13580 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: configured and running, proceeding with master startup.
16:14:50.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:50 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104887, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.66@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:50.666 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
16:14:50.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.672479 13604 catalog_manager.cc:820] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:14:50.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.675824 13108 heartbeater.cc:345] Connected to a master server at 127.12.47.125:38177
16:14:50.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.683632 13545 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "acc39da2179b470e918cf0f37a34954c" instance_seqno: 1795104878181549) as {username='kudu', principal='kudu/127.12.47.66@KRBTEST.COM'} at 127.12.47.66:41691; Asking this server to re-register.
16:14:50.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.685667 13108 heartbeater.cc:462] Registering TS with master...
16:14:50.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.690240 13544 ts_manager.cc:210] Registered new tserver with Master: acc39da2179b470e918cf0f37a34954c (127.12.47.66:39193)
16:14:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:50 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104890, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.125@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:50.722 [DEBUG - kudu-nio-0] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.12.47.126
16:14:50.723 [DEBUG - kudu-nio-2] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.12.47.125
16:14:50.726 [DEBUG - kudu-nio-1] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.12.47.124
16:14:50.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:50 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104890, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.125@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:50.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:50 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104890, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.125@KRBTEST.COM for kudu/127.12.47.124@KRBTEST.COM
16:14:50.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.810338 12959 heartbeater.cc:500] Master 127.12.47.126:33337 was elected leader, sending a full tablet report...
16:14:50.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:50 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104885, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.65@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:50.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.885326 12958 heartbeater.cc:345] Connected to a master server at 127.12.47.125:38177
16:14:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:50 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104884, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.126@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:50.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.889079 13545 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1c0bdd2da57349af93defb40ca49f821" instance_seqno: 1795104875481619) as {username='kudu', principal='kudu/127.12.47.65@KRBTEST.COM'} at 127.12.47.65:55995; Asking this server to re-register.
16:14:50.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.892025 12958 heartbeater.cc:462] Registering TS with master...
16:14:50.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.897324 13545 ts_manager.cc:210] Registered new tserver with Master: 1c0bdd2da57349af93defb40ca49f821 (127.12.47.65:37487)
16:14:50.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.924549 13555 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [term 1 FOLLOWER]: Advancing to term 2
16:14:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.965965 13587 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 127f4583c0704b96b6ee60f15dbd6a5f. Latest consensus state: current_term: 2 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:50.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.966678 13587 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: This master's current role is: FOLLOWER
16:14:50.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.968010 13614 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:50.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:50.968737 13614 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: This master's current role is: FOLLOWER
16:14:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:51 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1795104882, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:14:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:51 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1795104882, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:51.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:51 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1795104882, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.47.124@KRBTEST.COM
16:14:51.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:51.196563 13264 heartbeater.cc:500] Master 127.12.47.126:33337 was elected leader, sending a full tablet report...
16:14:51.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:51.202411 13611 negotiation.cc:336] Failed RPC negotiation. Trace:
16:14:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:14:51.184520 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.12.47.67:36083 (local address 127.12.47.125:38177)
16:14:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:14:51.185228 (+ 708us) server_negotiation.cc:207] Beginning negotiation
16:14:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:14:51.185257 (+ 29us) server_negotiation.cc:400] Waiting for connection header
16:14:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:14:51.191776 (+ 6519us) server_negotiation.cc:408] Connection header received
16:14:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:14:51.192267 (+ 491us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:14:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:14:51.192297 (+ 30us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:14:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:14:51.192577 (+ 280us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:14:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:14:51.192754 (+ 177us) server_negotiation.cc:227] Negotiated authn=SASL
16:14:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:14:51.202355 (+ 9601us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.12.47.67:36083: BlockingRecv error: recv got EOF from 127.12.47.67:36083 (error 108)
16:14:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":126}
16:14:51.247 [DEBUG - kudu-nio-1] (Negotiator.java:918) Authenticated connection [id: 0xec658748, L:/127.0.0.1:56374 - R:/127.12.47.124:46043] using SASL/GSSAPI
16:14:51.247 [DEBUG - kudu-nio-0] (Negotiator.java:918) Authenticated connection [id: 0x1f1d172e, L:/127.0.0.1:35344 - R:/127.12.47.126:33337] using SASL/GSSAPI
16:14:51.247 [DEBUG - kudu-nio-2] (Negotiator.java:918) Authenticated connection [id: 0xb5650421, L:/127.0.0.1:43544 - R:/127.12.47.125:38177] using SASL/GSSAPI
16:14:51.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:51 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104891, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:51.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:51.267357 13141 init.cc:295] Successfully reacquired a new kerberos TGT
16:14:51.292 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
16:14:51.292 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=980), Traces: [0ms] refreshing cache from master, [28ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.47.126:33337, [106ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.47.124:46043, [109ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.47.125:38177, [953ms] Sub RPC ConnectToMaster: received response from server master-127.12.47.125:38177: OK, [953ms] Sub RPC ConnectToMaster: received response from server master-127.12.47.124:46043: OK, [955ms] Sub RPC ConnectToMaster: received response from server master-127.12.47.126:33337: OK, deferred=null) after lookup
16:14:51.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:51.318076 13502 sys_catalog.cc:455] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:51.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:51.318892 13502 sys_catalog.cc:458] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: This master's current role is: LEADER
16:14:51.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:51.679872 13604 catalog_manager.cc:1310] Loaded cluster ID: 2dd45602808940128709b885b85b6703
16:14:51.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:51.680157 13604 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: loading cluster ID for follower catalog manager: success
16:14:51.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:51.685163 13604 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: acquiring CA information for follower catalog manager: success
16:14:51.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:51.689178 13604 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:14:52.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:52.216511 13259 heartbeater.cc:345] Connected to a master server at 127.12.47.125:38177
16:14:52.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:52.218963 13545 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac9f2cf9bc7543ad949581708041b41f" instance_seqno: 1795104881857195) as {username='kudu', principal='kudu/127.12.47.67@KRBTEST.COM'} at 127.12.47.67:60217; Asking this server to re-register.
16:14:52.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:52.220088 13259 heartbeater.cc:462] Registering TS with master...
16:14:52.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:52.222808 13545 ts_manager.cc:210] Registered new tserver with Master: ac9f2cf9bc7543ad949581708041b41f (127.12.47.67:43031)
16:14:52.316 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:14:52.317 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:14:52.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:52 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104892, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:52.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:14:52.356 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:52.356 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc
16:14:52.357 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:14:52.358 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.47.126:33337
16:14:52.358 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:14:52.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:52.358784 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 13272
16:14:52.390 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:52.390 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.47.124:46043
16:14:52.390 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:14:52.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:52.391083 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 13355
16:14:52.391 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.47.126:33337(127.12.47.126:33337)] handling channelInactive
16:14:52.392 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.47.126:33337(127.12.47.126:33337)] cleaning up while in state READY due to: connection closed
16:14:52.422 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.12.47.124:46043(127.12.47.124:46043)] handling channelInactive
16:14:52.423 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.47.124:46043(127.12.47.124:46043)] cleaning up while in state READY due to: connection closed
16:14:52.423 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:52.423 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.47.125:38177
16:14:52.424 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:14:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:52.424549 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 13464
16:14:52.450 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:52.450 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.47.126:33337
16:14:52.450 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.12.47.125:38177(127.12.47.125:38177)] handling channelInactive
16:14:52.451 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.12.47.125:38177(127.12.47.125:38177)] cleaning up while in state READY due to: connection closed
16:14:52.451 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:52.452765 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/logs
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/info.pb
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.126:33337
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.126
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37859
16:14:52.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.126
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:14:52.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.793738 13625 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:52.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.794379 13625 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:52.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.795033 13625 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.828709 13625 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:14:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.828938 13625 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.829228 13625 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.829603 13625 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:52.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.829846 13625 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:52.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.830051 13625 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:52.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.830214 13625 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42857/jwks.json
16:14:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:52.881240 13625 master_runner.cc:387] Master server non-default flags:
16:14:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/logs
16:14:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:52.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.126
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.126:33337
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/info.pb
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.126
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37859
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:52.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:52.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:52.882845 13625 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:52.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:52.884090 13625 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:52.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.899595 13633 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.903213 13636 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:52.900728 13634 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:54.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:54.296283 13635 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1393 milliseconds
16:14:54.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.296308 13625 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:54.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.297809 13625 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:54.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.302402 13625 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.303795 13625 hybrid_clock.cc:648] HybridClock initialized: now 1795104894303747 us; error 67 us; skew 500 ppm
16:14:54.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.304648 13625 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:54.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:54 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104894, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:54.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.311028 13625 init.cc:377] Logged in from keytab as kudu/127.12.47.126@KRBTEST.COM (short username kudu)
16:14:54.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.316468 13625 webserver.cc:577] Webserver started at http://127.12.47.126:37859/ using document root <none> and password file <none>
16:14:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.317322 13625 fs_manager.cc:362] Metadata directory not provided
16:14:54.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.317543 13625 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.325719 13625 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.004s sys 0.003s
16:14:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.330174 13649 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.331334 13625 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
16:14:54.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.331622 13625 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:54.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "127f4583c0704b96b6ee60f15dbd6a5f"
16:14:54.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:24 on dist-test-slave-4vz6"
16:14:54.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.333850 13625 fs_report.cc:389] FS layout report
16:14:54.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:54.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:54.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/wal
16:14:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/data
16:14:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:54.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:54.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.394205 13625 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:54.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.404795 13625 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:54.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.405212 13625 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:54.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.510774 13625 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.126:33337
16:14:54.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.510865 13701 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.126:33337 every 8 connection(s)
16:14:54.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.514662 13625 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-0/data/info.pb
16:14:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.524560 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 13625
16:14:54.525 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:54.525 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.47.124:46043
16:14:54.525 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:14:54.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.527462 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data
16:14:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:54.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:54.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/logs
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/info.pb
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.124:46043
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.124
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33711
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.124
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:54.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:14:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.528383 13702 sys_catalog.cc:263] Verifying existing consensus state
16:14:54.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.538924 13702 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Bootstrap starting.
16:14:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.587525 13702 log.cc:826] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Log is configured to *not* fsync() on all Append() calls
16:14:54.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.605684 13702 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:14:54.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.606552 13702 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Bootstrap complete.
16:14:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.628233 13702 raft_consensus.cc:360] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:54.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.630518 13702 raft_consensus.cc:748] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 127f4583c0704b96b6ee60f15dbd6a5f, State: Initialized, Role: FOLLOWER
16:14:54.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.631451 13702 consensus_queue.cc:261] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:54.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.634553 13706 sys_catalog.cc:455] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.635382 13706 sys_catalog.cc:458] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: This master's current role is: FOLLOWER
16:14:54.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.636330 13702 sys_catalog.cc:565] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: configured and running, proceeding with master startup.
16:14:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.670892 13717 catalog_manager.cc:1310] Loaded cluster ID: 2dd45602808940128709b885b85b6703
16:14:54.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.671227 13717 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: loading cluster ID for follower catalog manager: success
16:14:54.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.681077 13717 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: acquiring CA information for follower catalog manager: success
16:14:54.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.687393 13717 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:14:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:54 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104894, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.126@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.727993 13625 master_runner.cc:187] Error getting master registration for 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111),
16:14:54.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.754302 13625 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:14:54.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.841997 12959 heartbeater.cc:345] Connected to a master server at 127.12.47.126:33337
16:14:54.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.848279 13666 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1c0bdd2da57349af93defb40ca49f821" instance_seqno: 1795104875481619) as {username='kudu', principal='kudu/127.12.47.65@KRBTEST.COM'} at 127.12.47.65:47689; Asking this server to re-register.
16:14:54.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.849867 12959 heartbeater.cc:462] Registering TS with master...
16:14:54.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:54.853276 13666 ts_manager.cc:210] Registered new tserver with Master: 1c0bdd2da57349af93defb40ca49f821 (127.12.47.65:37487)
16:14:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:54.939632 13704 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:54.940335 13704 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:54.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:54.941121 13704 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:54.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:54.976131 13704 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:14:54.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:54.976441 13704 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:54.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:54.977147 13704 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:54.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:54.977546 13704 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:54.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:54.977942 13704 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:54.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:54.978292 13704 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:54.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:54.978544 13704 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42857/jwks.json
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.024616 13704 master_runner.cc:387] Master server non-default flags:
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/logs
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.124
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.124:46043
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:55.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:55.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:55.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:55.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:55.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:55.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:55.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:55.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/info.pb
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.124
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33711
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.026350 13704 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:55.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.027760 13704 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:55.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:55.040185 13727 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:55.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:55 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104895, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:55.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.244338 12839 init.cc:295] Successfully reacquired a new kerberos TGT
16:14:55.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.276037 13264 heartbeater.cc:345] Connected to a master server at 127.12.47.126:33337
16:14:55.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.279094 13666 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac9f2cf9bc7543ad949581708041b41f" instance_seqno: 1795104881857195) as {username='kudu', principal='kudu/127.12.47.67@KRBTEST.COM'} at 127.12.47.67:53375; Asking this server to re-register.
16:14:55.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.280802 13264 heartbeater.cc:462] Registering TS with master...
16:14:55.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.283907 13666 ts_manager.cc:210] Registered new tserver with Master: ac9f2cf9bc7543ad949581708041b41f (127.12.47.67:43031)
16:14:55.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:55.477216 11980 debug-util.cc:398] Leaking SignalData structure 0x7b080007e0c0 after lost signal to thread 11911
16:14:55.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:55.478282 11980 debug-util.cc:398] Leaking SignalData structure 0x7b080007e1a0 after lost signal to thread 11983
16:14:55.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.536999 13111 heartbeater.cc:345] Connected to a master server at 127.12.47.126:33337
16:14:55.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.540503 13666 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "acc39da2179b470e918cf0f37a34954c" instance_seqno: 1795104878181549) as {username='kudu', principal='kudu/127.12.47.66@KRBTEST.COM'} at 127.12.47.66:59209; Asking this server to re-register.
16:14:55.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.542136 13111 heartbeater.cc:462] Registering TS with master...
16:14:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:55.545334 13666 ts_manager.cc:210] Registered new tserver with Master: acc39da2179b470e918cf0f37a34954c (127.12.47.66:39193)
16:14:56.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:56.247110 13742 raft_consensus.cc:499] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:14:56.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:56.247722 13742 raft_consensus.cc:521] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:56.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:56.281153 13742 leader_election.cc:290] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177), c5f92d38aabf4855966756a73fad5f87 (127.12.47.124:46043)
16:14:56.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:56.283221 13653 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:56.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:56.293814 13653 leader_election.cc:336] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer c5f92d38aabf4855966756a73fad5f87 (127.12.47.124:46043): Network error: Client connection negotiation failed: client connection to 127.12.47.124:46043: connect: Connection refused (error 111)
16:14:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:56.304456 13650 leader_election.cc:336] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177): Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:56.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:56.305075 13650 leader_election.cc:304] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 127f4583c0704b96b6ee60f15dbd6a5f; no voters: 3567a5ad7a164c1cba57b85c36f6f434, c5f92d38aabf4855966756a73fad5f87
16:14:56.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:56.306149 13742 raft_consensus.cc:2771] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
16:14:57.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:56.442711 13726 debug-util.cc:398] Leaking SignalData structure 0x7b0800038620 after lost signal to thread 13704
16:14:57.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:57.234030 13726 kernel_stack_watchdog.cc:198] Thread 13704 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 398ms:
16:14:57.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:14:57.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:14:57.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:14:57.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:14:57.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:55.041504 13728 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:57.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:57.235786 13704 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 2.194s user 0.601s sys 1.056s
16:14:57.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:57.236351 13704 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 2.195s user 0.601s sys 1.057s
16:14:57.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:57.236729 13730 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:57.242806 13729 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2196 milliseconds
16:14:57.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.242826 13704 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:14:57.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.244544 13704 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:14:57.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.247303 13704 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:14:57.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.248687 13704 hybrid_clock.cc:648] HybridClock initialized: now 1795104897248644 us; error 50 us; skew 500 ppm
16:14:57.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.249737 13704 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:14:57.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:57 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104897, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:57.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.255884 13704 init.cc:377] Logged in from keytab as kudu/127.12.47.124@KRBTEST.COM (short username kudu)
16:14:57.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.261472 13704 webserver.cc:577] Webserver started at http://127.12.47.124:33711/ using document root <none> and password file <none>
16:14:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.262315 13704 fs_manager.cc:362] Metadata directory not provided
16:14:57.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.262499 13704 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:14:57.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.270895 13704 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.002s
16:14:57.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.277068 13756 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:14:57.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.285403 13704 fs_manager.cc:730] Time spent opening block manager: real 0.012s user 0.003s sys 0.000s
16:14:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.285825 13704 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c5f92d38aabf4855966756a73fad5f87"
16:14:57.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:29 on dist-test-slave-4vz6"
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.288558 13704 fs_report.cc:389] FS layout report
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/wal
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/data
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:14:57.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:14:57.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.449172 13704 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:14:57.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.457333 13704 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:14:57.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.458216 13704 kserver.cc:163] Server-wide thread pool size limit: 3276
16:14:57.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.603294 13704 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.124:46043
16:14:57.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.603335 13810 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.124:46043 every 8 connection(s)
16:14:57.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.607266 13704 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-2/data/info.pb
16:14:57.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.609596 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 13704
16:14:57.610 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:14:57.610 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.47.125:38177
16:14:57.611 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:14:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.614555 12477 external_mini_cluster.cc:1372] Running /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu
16:14:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data
16:14:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:14:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:14:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:14:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/logs
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/info.pb
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.125:38177
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.125
16:14:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36133
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.125
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:14:57.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:14:57.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.623935 13811 sys_catalog.cc:263] Verifying existing consensus state
16:14:57.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.636878 13811 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Bootstrap starting.
16:14:57.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.710389 13811 log.cc:826] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Log is configured to *not* fsync() on all Append() calls
16:14:57.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.752792 13811 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:14:57.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.753903 13811 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: Bootstrap complete.
16:14:57.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.792539 13811 raft_consensus.cc:360] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:57.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.796128 13811 raft_consensus.cc:748] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: c5f92d38aabf4855966756a73fad5f87, State: Initialized, Role: FOLLOWER
16:14:57.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.797416 13811 consensus_queue.cc:261] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:57.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.800706 13815 sys_catalog.cc:455] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:57.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.801693 13815 sys_catalog.cc:458] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: This master's current role is: FOLLOWER
16:14:57.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.802847 13811 sys_catalog.cc:565] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: configured and running, proceeding with master startup.
16:14:57.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.849790 13826 catalog_manager.cc:1310] Loaded cluster ID: 2dd45602808940128709b885b85b6703
16:14:57.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.850067 13826 catalog_manager.cc:1603] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: loading cluster ID for follower catalog manager: success
16:14:57.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.865051 13826 catalog_manager.cc:1625] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: acquiring CA information for follower catalog manager: success
16:14:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.871868 13826 catalog_manager.cc:1653] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:14:57.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:57 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104897, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.124@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:14:57.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:14:57 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104897, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:14:57.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.898264 12989 init.cc:295] Successfully reacquired a new kerberos TGT
16:14:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.899755 13704 master_runner.cc:187] Error getting master registration for 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111),
16:14:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:57.930159 13704 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:14:58.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.019042 12957 heartbeater.cc:345] Connected to a master server at 127.12.47.124:46043
16:14:58.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.028113 13775 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "1c0bdd2da57349af93defb40ca49f821" instance_seqno: 1795104875481619) as {username='kudu', principal='kudu/127.12.47.65@KRBTEST.COM'} at 127.12.47.65:46365; Asking this server to re-register.
16:14:58.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.035007 12957 heartbeater.cc:462] Registering TS with master...
16:14:58.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.038609 13775 ts_manager.cc:210] Registered new tserver with Master: 1c0bdd2da57349af93defb40ca49f821 (127.12.47.65:37487)
16:14:58.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.165751 13813 flags.cc:432] Enabled unsafe flag: --openssl_security_level_override=0
16:14:58.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.166453 13813 flags.cc:432] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:14:58.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.167466 13813 flags.cc:432] Enabled unsafe flag: --never_fsync=true
16:14:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.207216 13813 flags.cc:432] Enabled experimental flag: --authn_token_validity_seconds=10
16:14:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.207508 13813 flags.cc:432] Enabled experimental flag: --ipki_ca_key_size=768
16:14:58.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.207937 13813 flags.cc:432] Enabled experimental flag: --ipki_server_key_size=768
16:14:58.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.208303 13813 flags.cc:432] Enabled experimental flag: --tsk_num_rsa_bits=512
16:14:58.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.208647 13813 flags.cc:432] Enabled experimental flag: --rpc_reuseport=true
16:14:58.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.208987 13813 flags.cc:432] Enabled experimental flag: --enable_jwt_token_auth=true
16:14:58.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.209246 13813 flags.cc:432] Enabled experimental flag: --jwks_url=http://localhost:42857/jwks.json
16:14:58.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.250978 13813 master_runner.cc:387] Master server non-default flags:
16:14:58.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/logs
16:14:58.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:14:58.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:14:58.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:14:58.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.47.84:44421
16:14:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:14:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:14:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data
16:14:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:14:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:14:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:14:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.47.126:33337,127.12.47.125:38177,127.12.47.124:46043
16:14:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/krb5kdc/kudu.keytab
16:14:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.47.125
16:14:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:14:58.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.47.125:38177
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42857/jwks.json
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/info.pb
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.47.125
16:14:58.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=36133
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision bb7a73e1ed87d3ff0c1acf8506c87b28983a7cff
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 19 Nov 2026 15:45:50 UTC on bdcb31816ec0
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 14862
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.252568 13813 env_posix.cc:2337] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:14:58.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.254168 13813 file_cache.cc:504] Constructed file cache file cache with capacity 419430
16:14:58.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.265861 13841 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:14:58.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.322243 13258 heartbeater.cc:345] Connected to a master server at 127.12.47.124:46043
16:14:58.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.325991 13775 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "ac9f2cf9bc7543ad949581708041b41f" instance_seqno: 1795104881857195) as {username='kudu', principal='kudu/127.12.47.67@KRBTEST.COM'} at 127.12.47.67:59929; Asking this server to re-register.
16:14:58.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.329927 13258 heartbeater.cc:462] Registering TS with master...
16:14:58.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.334033 13775 ts_manager.cc:210] Registered new tserver with Master: ac9f2cf9bc7543ad949581708041b41f (127.12.47.67:43031)
16:14:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.364531 13254 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4000 after lost signal to thread 13124
16:14:58.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.366221 13254 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae4a0 after lost signal to thread 13257
16:14:58.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.412918 13103 debug-util.cc:398] Leaking SignalData structure 0x7b08000b4060 after lost signal to thread 12973
16:14:58.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.414033 13103 debug-util.cc:398] Leaking SignalData structure 0x7b08000b7a80 after lost signal to thread 13106
16:14:58.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.426314 13848 raft_consensus.cc:499] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:14:58.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.427384 13848 raft_consensus.cc:521] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:58.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.434516 13848 leader_election.cc:290] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177), c5f92d38aabf4855966756a73fad5f87 (127.12.47.124:46043)
16:14:58.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.496665 13650 leader_election.cc:336] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177): Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:58.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.501956 13785 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "c5f92d38aabf4855966756a73fad5f87" is_pre_election: true
16:14:58.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.502928 13785 raft_consensus.cc:2490] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 127f4583c0704b96b6ee60f15dbd6a5f in term 2.
16:14:58.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.504369 13653 leader_election.cc:304] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 127f4583c0704b96b6ee60f15dbd6a5f, c5f92d38aabf4855966756a73fad5f87; no voters: 3567a5ad7a164c1cba57b85c36f6f434
16:14:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.505177 13848 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Leader pre-election won for term 3
16:14:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.505548 13848 raft_consensus.cc:499] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:14:58.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.505836 13848 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 2 FOLLOWER]: Advancing to term 3
16:14:58.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.534709 13848 raft_consensus.cc:521] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:58.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.537577 13848 leader_election.cc:290] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 3 election: Requested vote from peers 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177), c5f92d38aabf4855966756a73fad5f87 (127.12.47.124:46043)
16:14:58.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.541167 13785 tablet_service.cc:1901] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "c5f92d38aabf4855966756a73fad5f87"
16:14:58.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.541774 13785 raft_consensus.cc:3096] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 2 FOLLOWER]: Advancing to term 3
16:14:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.546533 13650 leader_election.cc:336] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177): Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111)
16:14:58.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.554704 13785 raft_consensus.cc:2490] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 127f4583c0704b96b6ee60f15dbd6a5f in term 3.
16:14:58.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.556143 13653 leader_election.cc:304] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 127f4583c0704b96b6ee60f15dbd6a5f, c5f92d38aabf4855966756a73fad5f87; no voters: 3567a5ad7a164c1cba57b85c36f6f434
16:14:58.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.558013 13848 raft_consensus.cc:2826] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 3 FOLLOWER]: Leader election won for term 3
16:14:58.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.572201 13848 raft_consensus.cc:705] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [term 3 LEADER]: Becoming Leader. State: Replica: 127f4583c0704b96b6ee60f15dbd6a5f, State: Running, Role: LEADER
16:14:58.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.573511 13848 consensus_queue.cc:238] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:14:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.602458 13852 sys_catalog.cc:455] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 127f4583c0704b96b6ee60f15dbd6a5f. Latest consensus state: current_term: 3 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.604628 13852 sys_catalog.cc:458] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: This master's current role is: LEADER
16:14:58.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.618319 13785 raft_consensus.cc:1285] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [term 3 FOLLOWER]: Refusing update from remote peer 127f4583c0704b96b6ee60f15dbd6a5f: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
16:14:58.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.623255 13848 consensus_queue.cc:1049] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [LEADER]: Connected to new peer: Peer: permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
16:14:58.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.628705 13650 consensus_peers.cc:597] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f -> Peer 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177): Couldn't send request to peer 3567a5ad7a164c1cba57b85c36f6f434. Status: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
16:14:58.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.614697 13856 catalog_manager.cc:1526] Loading table and tablet metadata into memory...
16:14:58.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.643425 13107 heartbeater.cc:345] Connected to a master server at 127.12.47.124:46043
16:14:58.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.664109 13856 catalog_manager.cc:1535] Initializing Kudu cluster ID...
16:14:58.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.691186 13857 sys_catalog.cc:455] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 127f4583c0704b96b6ee60f15dbd6a5f. Latest consensus state: current_term: 3 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.694222 13857 sys_catalog.cc:458] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: This master's current role is: FOLLOWER
16:14:58.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.693183 13775 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "acc39da2179b470e918cf0f37a34954c" instance_seqno: 1795104878181549) as {username='kudu', principal='kudu/127.12.47.66@KRBTEST.COM'} at 127.12.47.66:34893; Asking this server to re-register.
16:14:58.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.696591 13107 heartbeater.cc:462] Registering TS with master...
16:14:58.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.702226 11900 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5e00 after lost signal to thread 11835
16:14:58.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.703917 11900 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6c20 after lost signal to thread 11903
16:14:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.698578 13852 sys_catalog.cc:455] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:58.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.701292 13852 sys_catalog.cc:458] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f [sys.catalog]: This master's current role is: LEADER
16:14:58.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.730948 13856 catalog_manager.cc:1310] Loaded cluster ID: 2dd45602808940128709b885b85b6703
16:14:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.736342 13856 catalog_manager.cc:1546] Initializing Kudu internal certificate authority...
16:14:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.739120 13857 sys_catalog.cc:455] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:14:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.739847 13857 sys_catalog.cc:458] T 00000000000000000000000000000000 P c5f92d38aabf4855966756a73fad5f87 [sys.catalog]: This master's current role is: FOLLOWER
16:14:58.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.740053 13775 ts_manager.cc:210] Registered new tserver with Master: acc39da2179b470e918cf0f37a34954c (127.12.47.66:39193)
16:14:58.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.757879 13856 catalog_manager.cc:1555] Loading token signing keys...
16:14:58.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.789956 13856 catalog_manager.cc:6248] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f: Loaded TSK: 0
16:14:58.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:58.792083 13856 catalog_manager.cc:1565] Initializing in-progress tserver states...
16:14:59.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:59.762223 13625 master_runner.cc:187] Error getting master registration for 127.12.47.125:38177: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111),
16:14:59.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:59.762822 13625 master_runner.cc:419] Couldn't verify the masters in the cluster. Trying again...
16:14:59.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:14:59.874814 12959 heartbeater.cc:500] Master 127.12.47.126:33337 was elected leader, sending a full tablet report...
16:15:00.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:00.491730 13264 heartbeater.cc:500] Master 127.12.47.126:33337 was elected leader, sending a full tablet report...
16:15:00.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:00.621341 13111 heartbeater.cc:500] Master 127.12.47.126:33337 was elected leader, sending a full tablet report...
16:15:01.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:59.667510 13840 debug-util.cc:398] Leaking SignalData structure 0x7b0800038620 after lost signal to thread 13813
16:15:01.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:15:01.071241 13840 kernel_stack_watchdog.cc:198] Thread 13813 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:643 for 397ms:
16:15:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:15:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:15:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:15:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:15:01.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:14:58.267194 13842 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:15:01.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:15:01.079140 13813 thread.cc:642] OpenStack (cloud detector) Time spent creating pthread: real 2.811s user 0.000s sys 0.005s
16:15:01.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:15:01.079515 13813 thread.cc:609] OpenStack (cloud detector) Time spent starting thread: real 2.811s user 0.000s sys 0.005s
16:15:01.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.079947 13813 server_base.cc:1061] running on GCE node
16:15:01.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:15:01.081053 13844 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:15:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.082644 13813 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:15:01.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.089811 13813 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:15:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.091274 13813 hybrid_clock.cc:648] HybridClock initialized: now 1795104901091237 us; error 45 us; skew 500 ppm
16:15:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.092523 13813 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:15:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:15:01 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104901, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:15:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.101711 13813 init.cc:377] Logged in from keytab as kudu/127.12.47.125@KRBTEST.COM (short username kudu)
16:15:01.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.125725 13813 webserver.cc:577] Webserver started at http://127.12.47.125:36133/ using document root <none> and password file <none>
16:15:01.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.127331 13813 fs_manager.cc:362] Metadata directory not provided
16:15:01.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.127815 13813 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:15:01.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.145978 13813 fs_manager.cc:714] Time spent opening directory manager: real 0.014s user 0.009s sys 0.004s
16:15:01.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.159111 13879 log_block_manager.cc:4130] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:15:01.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.160995 13813 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.000s sys 0.007s
16:15:01.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.162138 13813 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data,/tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:15:01.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3567a5ad7a164c1cba57b85c36f6f434"
16:15:01.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-11-19 16:14:27 on dist-test-slave-4vz6"
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.165323 13813 fs_report.cc:389] FS layout report
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/wal
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/data
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:15:01.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:15:01.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:15:01.175745 13650 consensus_peers.cc:597] T 00000000000000000000000000000000 P 127f4583c0704b96b6ee60f15dbd6a5f -> Peer 3567a5ad7a164c1cba57b85c36f6f434 (127.12.47.125:38177): Couldn't send request to peer 3567a5ad7a164c1cba57b85c36f6f434. Status: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
16:15:01.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:15:01 dist-test-slave-4vz6 krb5kdc[12489](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104901, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:15:01.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.272294 13141 init.cc:295] Successfully reacquired a new kerberos TGT
16:15:01.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.321623 13813 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:15:01.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.334192 13813 env_posix.cc:2337] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:15:01.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.334777 13813 kserver.cc:163] Server-wide thread pool size limit: 3276
16:15:01.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:15:01.568526 13650 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.47.125:38177: connect: Connection refused (error 111) [suppressed 9 similar messages]
16:15:01.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.830601 13934 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.47.125:38177 every 8 connection(s)
16:15:01.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.831272 13813 rpc_server.cc:307] RPC server started. Bound to: 127.12.47.125:38177
16:15:01.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.836098 13813 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskwnt6fz/test-tmp/mini-kudu-cluster11673267927886584652/master-1/data/info.pb
16:15:01.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.843600 12477 external_mini_cluster.cc:1434] Started /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu as pid 13813
16:15:01.845 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:15:01.846 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.47.126:33337(127.12.47.126:33337)] connecting to peer
16:15:01.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.865077 13935 sys_catalog.cc:263] Verifying existing consensus state
16:15:01.874 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.12.47.126:33337(127.12.47.126:33337)] Successfully connected to peer
16:15:01.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.873697 13935 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Bootstrap starting.
16:15:01.907 [DEBUG - kudu-nio-3] (Negotiator.java:528) SASL mechanism GSSAPI chosen for peer 127.12.47.126
16:15:02.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:01.946316 13935 log.cc:826] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Log is configured to *not* fsync() on all Append() calls
16:15:02.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:15:01 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104894, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.126@KRBTEST.COM for kudu/127.12.47.125@KRBTEST.COM
16:15:02.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.016557 13935 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:15:02.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.017585 13935 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: Bootstrap complete.
16:15:02.043 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.12.47.126:33337(127.12.47.126:33337)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
16:15:02.046 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.47.125:38177(127.12.47.125:38177)] disconnecting while in state TERMINATED
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:15:02.045972 13938 negotiation.cc:336] Failed RPC negotiation. Trace:
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:01.874627 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.0.0.1:49716 (local address 127.12.47.126:33337)
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:01.894058 (+ 19431us) server_negotiation.cc:207] Beginning negotiation
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:01.895741 (+ 1683us) server_negotiation.cc:400] Waiting for connection header
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:01.895800 (+ 59us) server_negotiation.cc:408] Connection header received
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:01.895948 (+ 148us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:01.895982 (+ 34us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:01.896241 (+ 259us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:01.896460 (+ 219us) server_negotiation.cc:227] Negotiated authn=SASL
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:02.012595 (+116135us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:02.014209 (+ 1614us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
16:15:02.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:02.029271 (+ 15062us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
16:15:02.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:02.029975 (+ 704us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
16:15:02.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:02.045915 (+ 15940us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49716: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:49716): Cannot send after transport endpoint shutdown (error 108)
16:15:02.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":18215,"spinlock_wait_cycles":387840,"thread_start_us":18289,"threads_started":1}
16:15:02.048 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.47.124:46043(127.12.47.124:46043)] disconnecting while in state TERMINATED
16:15:02.049 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.47.126:33337(127.12.47.126:33337)] disconnecting while in state TERMINATED
16:15:02.060 [DEBUG - kudu-nio-3] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:15:02.066 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.12.47.126:33337(127.12.47.126:33337)] handling channelInactive
16:15:02.066 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:15:02.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.105769 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 12819
16:15:02.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.113025 13935 raft_consensus.cc:360] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:15:02.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.130330 13935 raft_consensus.cc:748] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3567a5ad7a164c1cba57b85c36f6f434, State: Initialized, Role: FOLLOWER
16:15:02.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.135422 13935 consensus_queue.cc:261] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } }
16:15:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.166114 13949 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:15:02.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.166940 13949 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: This master's current role is: FOLLOWER
16:15:02.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.170010 13935 sys_catalog.cc:565] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: configured and running, proceeding with master startup.
16:15:02.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.171479 13909 raft_consensus.cc:3096] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [term 2 FOLLOWER]: Advancing to term 3
16:15:02.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:15:02.242594 13940 negotiation.cc:336] Failed RPC negotiation. Trace:
16:15:02.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:02.083779 (+ 0us) reactor.cc:730] Submitting negotiation task for server connection from 127.12.47.65:53363 (local address 127.12.47.125:38177)
16:15:02.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:02.084599 (+ 820us) server_negotiation.cc:207] Beginning negotiation
16:15:02.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:02.084624 (+ 25us) server_negotiation.cc:400] Waiting for connection header
16:15:02.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1119 16:15:02.242404 (+157780us) negotiation.cc:326] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.12.47.65:53363: BlockingRecv error: recv got EOF from 127.12.47.65:53363 (error 108)
16:15:02.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":219}
16:15:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.250381 13949 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 127f4583c0704b96b6ee60f15dbd6a5f. Latest consensus state: current_term: 3 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:15:02.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.251180 13949 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: This master's current role is: FOLLOWER
16:15:02.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.253443 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 12972
16:15:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.252463 13957 sys_catalog.cc:455] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "127f4583c0704b96b6ee60f15dbd6a5f" member_type: VOTER last_known_addr { host: "127.12.47.126" port: 33337 } } peers { permanent_uuid: "3567a5ad7a164c1cba57b85c36f6f434" member_type: VOTER last_known_addr { host: "127.12.47.125" port: 38177 } } peers { permanent_uuid: "c5f92d38aabf4855966756a73fad5f87" member_type: VOTER last_known_addr { host: "127.12.47.124" port: 46043 } } }
16:15:02.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.256242 13957 sys_catalog.cc:458] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434 [sys.catalog]: This master's current role is: FOLLOWER
16:15:02.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.307061 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 13121
16:15:02.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.337520 13963 catalog_manager.cc:1310] Loaded cluster ID: 2dd45602808940128709b885b85b6703
16:15:02.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.337828 13963 catalog_manager.cc:1603] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: loading cluster ID for follower catalog manager: success
16:15:02.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.346735 13963 catalog_manager.cc:1625] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: acquiring CA information for follower catalog manager: success
16:15:02.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.349677 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 13625
16:15:02.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Nov 19 16:15:02 dist-test-slave-4vz6 krb5kdc[12489](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1795104901, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.47.125@KRBTEST.COM for kudu/127.12.47.126@KRBTEST.COM
16:15:02.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.353061 13963 catalog_manager.cc:1653] T 00000000000000000000000000000000 P 3567a5ad7a164c1cba57b85c36f6f434: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:15:02.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.398674 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 13813
16:15:02.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:02.424387 12477 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 13704
16:15:03.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:15:03.592188 11825 debug-util.cc:398] Leaking SignalData structure 0x7b080007a2c0 after lost signal to thread 11762
16:15:03.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20261119 16:15:03.592957 11825 debug-util.cc:398] Leaking SignalData structure 0x7b080008a2e0 after lost signal to thread 11828
16:15:03.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-19T16:15:03Z chronyd exiting
16:15:04.693 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:15:04.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:04.696157 11749 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 12031
16:15:04.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:04.724161 11749 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 12191
16:15:04.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:04.760224 11749 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 12331
16:15:04.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:04.789340 11749 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 11761
16:15:04.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:04.822971 11749 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 11831
16:15:04.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20261119 16:15:04.854745 11749 external_mini_cluster.cc:1664] Killing /tmp/dist-test-taskwnt6fz/build/tsan/bin/kudu with pid 11906
16:15:06.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-11-19T16:15:06Z chronyd exiting
16:15:07.061 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:988)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:844)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:781)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:730)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:378)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: javax.security.sasl.SaslException: GSS initiate failed
at jdk.security.jgss/com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:228)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:958)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:955)
at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.apache.kudu.client.internals.ReflectiveStrategy.invokeChecked(ReflectiveStrategy.java:48)
at org.apache.kudu.client.internals.LegacyStrategy.doAs(LegacyStrategy.java:94)
at org.apache.kudu.client.internals.LegacyStrategy.callAs(LegacyStrategy.java:102)
at org.apache.kudu.client.internals.CompositeStrategy.lambda$callAs$1(CompositeStrategy.java:113)
at org.apache.kudu.client.internals.CompositeStrategy.performAction(CompositeStrategy.java:84)
at org.apache.kudu.client.internals.CompositeStrategy.callAs(CompositeStrategy.java:113)
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:954)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:844)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:781)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:730)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:378)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at java.security.jgss/sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:166)
at java.security.jgss/sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:126)
at java.security.jgss/sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:195)
at java.security.jgss/sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:205)
at java.security.jgss/sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:230)
at java.security.jgss/sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:196)
at jdk.security.jgss/com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:209)
... 52 more
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:988)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:844)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:781)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:730)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:378)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
at jdk.security.jgss/com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:228)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:958)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:955)
at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.apache.kudu.client.internals.ReflectiveStrategy.invokeChecked(ReflectiveStrategy.java:48)
at org.apache.kudu.client.internals.LegacyStrategy.doAs(LegacyStrategy.java:94)
at org.apache.kudu.client.internals.LegacyStrategy.callAs(LegacyStrategy.java:102)
at org.apache.kudu.client.internals.CompositeStrategy.lambda$callAs$1(CompositeStrategy.java:113)
at org.apache.kudu.client.internals.CompositeStrategy.performAction(CompositeStrategy.java:84)
at org.apache.kudu.client.internals.CompositeStrategy.callAs(CompositeStrategy.java:113)
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:954)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:844)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:781)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:730)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:378)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at java.security.jgss/sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:166)
at java.security.jgss/sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:126)
at java.security.jgss/sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:195)
at java.security.jgss/sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:205)
at java.security.jgss/sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:230)
at java.security.jgss/sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:196)
at jdk.security.jgss/com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:209)
... 52 more
16:15:07.759 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
16:15:07.759 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:988)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:844)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:781)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:730)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:378)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: javax.security.sasl.SaslException: GSS initiate failed
at jdk.security.jgss/com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:228)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:958)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:955)
at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.apache.kudu.client.internals.ReflectiveStrategy.invokeChecked(ReflectiveStrategy.java:48)
at org.apache.kudu.client.internals.LegacyStrategy.doAs(LegacyStrategy.java:94)
at org.apache.kudu.client.internals.LegacyStrategy.callAs(LegacyStrategy.java:102)
at org.apache.kudu.client.internals.CompositeStrategy.lambda$callAs$1(CompositeStrategy.java:113)
at org.apache.kudu.client.internals.CompositeStrategy.performAction(CompositeStrategy.java:84)
at org.apache.kudu.client.internals.CompositeStrategy.callAs(CompositeStrategy.java:113)
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:954)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:844)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:781)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:730)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:378)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: org.ietf.jgss.GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at java.security.jgss/sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:166)
at java.security.jgss/sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:126)
at java.security.jgss/sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:195)
at java.security.jgss/sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:205)
at java.security.jgss/sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:230)
at java.security.jgss/sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:196)
at jdk.security.jgss/com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:209)
... 52 more
org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:160)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:456)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:988)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:844)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:781)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:730)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:378)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]
at jdk.security.jgss/com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:228)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:958)
at org.apache.kudu.client.Negotiator$1.call(Negotiator.java:955)
at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.apache.kudu.client.internals.ReflectiveStrategy.invokeChecked(ReflectiveStrategy.java:48)
at org.apache.kudu.client.internals.LegacyStrategy.doAs(LegacyStrategy.java:94)
at org.apache.kudu.client.internals.LegacyStrategy.callAs(LegacyStrategy.java:102)
at org.apache.kudu.client.internals.CompositeStrategy.lambda$callAs$1(CompositeStrategy.java:113)
at org.apache.kudu.client.internals.CompositeStrategy.performAction(CompositeStrategy.java:84)
at org.apache.kudu.client.internals.CompositeStrategy.callAs(CompositeStrategy.java:113)
at org.apache.kudu.client.Negotiator.evaluateChallenge(Negotiator.java:954)
at org.apache.kudu.client.Negotiator.sendSaslInitiate(Negotiator.java:844)
at org.apache.kudu.client.Negotiator.startAuthentication(Negotiator.java:781)
at org.apache.kudu.client.Negotiator.handleTlsMessage(Negotiator.java:730)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:378)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:93)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1399)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:910)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:1007)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)
at java.security.jgss/sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:166)
at java.security.jgss/sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:126)
at java.security.jgss/sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:195)
at java.security.jgss/sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:205)
at java.security.jgss/sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:230)
at java.security.jgss/sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:196)
at jdk.security.jgss/com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:209)
... 52 more