16:45:41.744 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
16:45:41.756 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskXsbxVh/test-tmp
16:45:41.759 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:45:41.762 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskXsbxVh/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:45:42.069 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
16:45:42.195 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:45:42.196 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:45:42.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:42.200089 29176 mini_oidc.cc:150] Starting JWKS server
16:45:42.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:42.203773 29176 webserver.cc:492] Webserver started at http://127.0.0.1:37013/ using document root <none> and password file <none>
16:45:42.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:42.204216 29176 mini_oidc.cc:186] Starting OIDC Discovery server
16:45:42.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:42.205938 29176 webserver.cc:492] Webserver started at http://127.0.0.1:42751/ using document root <none> and password file <none>
16:45:42.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T16:45:42Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
16:45:42.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T16:45:42Z Disabled control of system clock
16:45:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:42.268831 29176 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:42.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:42.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/wal
16:45:42.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/data
16:45:42.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:45:42.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:45:42.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:45:42.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:42.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:42.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:42.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/logs
16:45:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/data/info.pb
16:45:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:45:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:45:42.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:45:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:45:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:45:42.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:45:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:45:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:45:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.62:44765
16:45:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.62
16:45:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:42.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:42.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:42.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:42.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:45:42.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:45:42.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:42.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:45:42.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37013/jwks.json with env {}
16:45:42.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:42.574234 29188 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:45:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:42.574795 29188 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:45:42.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:42.575460 29188 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:45:42.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:42.608377 29188 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:45:42.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:42.608675 29188 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:45:42.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:42.608879 29188 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:45:42.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:42.609063 29188 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:45:42.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:42.609302 29188 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:45:42.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:42.609546 29188 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37013/jwks.json
16:45:42.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:42.647053 29188 master_runner.cc:386] Master server non-default flags:
16:45:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:45:42.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/data
16:45:42.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/wal
16:45:42.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:45:42.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:45:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.62:44765
16:45:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:45:42.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:45:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:45:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37013/jwks.json
16:45:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:42.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/data/info.pb
16:45:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.62
16:45:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:45:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:45:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:45:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:42.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/logs
16:45:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:45:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:45:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:45:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:45:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:45:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:45:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:45:42.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:45:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:45:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:42.648720 29188 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:45:42.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:42.650929 29188 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:45:42.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:42.664816 29195 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:44.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:42.664866 29194 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:44.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.067391 29193 debug-util.cc:398] Leaking SignalData structure 0x7b0800037520 after lost signal to thread 29188
16:45:44.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.488837 29193 kernel_stack_watchdog.cc:198] Thread 29188 stuck at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:642 for 400ms:
16:45:44.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:45:44.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:45:44.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:45:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:45:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.489145 29188 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.825s user 0.586s sys 1.231s
16:45:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.489460 29188 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.826s user 0.586s sys 1.231s
16:45:44.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.490327 29197 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.492858 29196 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1824 milliseconds
16:45:44.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.492944 29188 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:45:44.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.494262 29188 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:45:44.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.496793 29188 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:45:44.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.498145 29188 hybrid_clock.cc:648] HybridClock initialized: now 1761756344498113 us; error 44 us; skew 500 ppm
16:45:44.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.499012 29188 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:45:44.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.505151 29188 webserver.cc:492] Webserver started at http://127.28.126.62:42703/ using document root <none> and password file <none>
16:45:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.506143 29188 fs_manager.cc:362] Metadata directory not provided
16:45:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.506366 29188 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:45:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.506891 29188 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:45:44.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.511279 29188 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/data/instance:
16:45:44.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3c1e3bcc76842f997ef6a4789353b8d"
16:45:44.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:44 on dist-test-slave-wkbt"
16:45:44.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.512629 29188 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/wal/instance:
16:45:44.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3c1e3bcc76842f997ef6a4789353b8d"
16:45:44.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:44 on dist-test-slave-wkbt"
16:45:44.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.520448 29188 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
16:45:44.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.526329 29204 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:45:44.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.527506 29188 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
16:45:44.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.527866 29188 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/wal
16:45:44.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "e3c1e3bcc76842f997ef6a4789353b8d"
16:45:44.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:44 on dist-test-slave-wkbt"
16:45:44.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.528302 29188 fs_report.cc:389] FS layout report
16:45:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:45:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/wal
16:45:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/wal
16:45:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/data/data
16:45:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:45:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:45:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:45:44.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:45:44.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:45:44.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:45:44.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:45:44.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:45:44.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:45:44.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:45:44.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:45:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.579231 29188 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:45:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.580873 29188 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:45:44.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.581405 29188 kserver.cc:163] Server-wide thread pool size limit: 3276
16:45:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.653153 29188 rpc_server.cc:307] RPC server started. Bound to: 127.28.126.62:44765
16:45:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.653234 29255 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.126.62:44765 every 8 connection(s)
16:45:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.656111 29188 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/data/info.pb
16:45:44.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.662417 29256 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:45:44.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.666023 29176 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 29188
16:45:44.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.666568 29176 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-0/wal/instance
16:45:44.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.671911 29176 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:44.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:44.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/wal
16:45:44.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/data
16:45:44.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/logs
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/data/info.pb
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:45:44.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:45:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:44.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:44.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:44.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:45:44.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:45:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:45:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:45:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.61:45025
16:45:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.61
16:45:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:45:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:45:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:45:44.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37013/jwks.json with env {}
16:45:44.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.678903 29256 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } has no permanent_uuid. Determining permanent_uuid...
16:45:44.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.710294 29256 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } has no permanent_uuid. Determining permanent_uuid...
16:45:44.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.714370 29208 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.126.61:45025: connect: Connection refused (error 111)
16:45:44.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.718815 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.61:45025: Network error: Client connection negotiation failed: client connection to 127.28.126.61:45025: connect: Connection refused (error 111)
16:45:44.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.768499 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } attempt: 1
16:45:44.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.772723 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.61:45025: Network error: Client connection negotiation failed: client connection to 127.28.126.61:45025: connect: Connection refused (error 111)
16:45:44.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.841266 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } attempt: 2
16:45:44.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.845330 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.61:45025: Network error: Client connection negotiation failed: client connection to 127.28.126.61:45025: connect: Connection refused (error 111)
16:45:44.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:44.936873 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } attempt: 3
16:45:44.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.940955 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.61:45025: Network error: Client connection negotiation failed: client connection to 127.28.126.61:45025: connect: Connection refused (error 111)
16:45:44.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.987912 29258 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:45:44.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.988474 29258 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:45:44.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:44.988946 29258 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:45:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:45.021241 29258 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:45:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:45.021541 29258 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:45:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:45.021734 29258 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:45:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:45.021929 29258 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:45:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:45.022120 29258 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:45:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:45.022266 29258 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37013/jwks.json
16:45:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:45.059810 29258 master_runner.cc:386] Master server non-default flags:
16:45:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:45:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/data
16:45:45.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/wal
16:45:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:45:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:45:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.61:45025
16:45:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:45:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:45:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:45:45.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37013/jwks.json
16:45:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/data/info.pb
16:45:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.61
16:45:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:45:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:45:45.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/logs
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:45.061271 29258 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:45:45.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:45.063544 29258 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:45:45.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:45.074045 29267 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:45.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:45.084475 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } attempt: 4
16:45:45.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:45.091677 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.61:45025: Network error: Client connection negotiation failed: client connection to 127.28.126.61:45025: connect: Connection refused (error 111)
16:45:45.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:45.391256 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } attempt: 5
16:45:45.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:45.398272 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.61:45025: Network error: Client connection negotiation failed: client connection to 127.28.126.61:45025: connect: Connection refused (error 111)
16:45:45.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:45.945863 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } attempt: 6
16:45:45.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:45.959046 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.61:45025: Network error: Client connection negotiation failed: client connection to 127.28.126.61:45025: connect: Connection refused (error 111)
16:45:46.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:45.076601 29268 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:46.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.278921 29269 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:45:46.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.280527 29270 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:46.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.283932 29258 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.209s user 0.462s sys 0.746s
16:45:46.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.284209 29258 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.209s user 0.462s sys 0.746s
16:45:46.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.284482 29258 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:45:46.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.285686 29258 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:45:46.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.288511 29258 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:45:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.289912 29258 hybrid_clock.cc:648] HybridClock initialized: now 1761756346289820 us; error 87 us; skew 500 ppm
16:45:46.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.290746 29258 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:45:46.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.297927 29258 webserver.cc:492] Webserver started at http://127.28.126.61:40893/ using document root <none> and password file <none>
16:45:46.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.299042 29258 fs_manager.cc:362] Metadata directory not provided
16:45:46.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.299309 29258 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:45:46.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.299949 29258 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:45:46.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.304348 29258 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/data/instance:
16:45:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4df8813fc42646d9bb9369ea803d66d3"
16:45:46.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:46 on dist-test-slave-wkbt"
16:45:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.305747 29258 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/wal/instance:
16:45:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4df8813fc42646d9bb9369ea803d66d3"
16:45:46.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:46 on dist-test-slave-wkbt"
16:45:46.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.316373 29258 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.010s sys 0.002s
16:45:46.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.323032 29279 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:45:46.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.324201 29258 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
16:45:46.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.324474 29258 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/wal
16:45:46.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4df8813fc42646d9bb9369ea803d66d3"
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:46 on dist-test-slave-wkbt"
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.324788 29258 fs_report.cc:389] FS layout report
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/wal
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/wal
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/data/data
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:45:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:45:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:45:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:45:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:45:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:45:46.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:45:46.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.410009 29258 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:45:46.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.411495 29258 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:45:46.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.411972 29258 kserver.cc:163] Server-wide thread pool size limit: 3276
16:45:46.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.496505 29258 rpc_server.cc:307] RPC server started. Bound to: 127.28.126.61:45025
16:45:46.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.496652 29330 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.126.61:45025 every 8 connection(s)
16:45:46.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.499239 29258 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/data/info.pb
16:45:46.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.504022 29176 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 29258
16:45:46.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.504370 29176 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-1/wal/instance
16:45:46.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.506023 29331 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:45:46.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.506357 29176 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:46.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:46.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/wal
16:45:46.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/data
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/logs
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/data/info.pb
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:45:46.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:46.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:46.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:46.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:45:46.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:45:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:45:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:45:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.60:45375
16:45:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.60
16:45:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:45:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:45:46.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:45:46.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37013/jwks.json with env {}
16:45:46.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.522047 29331 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } has no permanent_uuid. Determining permanent_uuid...
16:45:46.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.544157 29331 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } has no permanent_uuid. Determining permanent_uuid...
16:45:46.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.563699 29331 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } has no permanent_uuid. Determining permanent_uuid...
16:45:46.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.566087 29281 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:46.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.574916 29331 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:46.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.624555 29331 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 1
16:45:46.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.629676 29331 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:46.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.698207 29331 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 2
16:45:46.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.703697 29331 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:46.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.795095 29331 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 3
16:45:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.800673 29331 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:46.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.941205 29333 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:45:46.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.942305 29333 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:45:46.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.943418 29333 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:45:46.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:46.944172 29331 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 4
16:45:46.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.951195 29331 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:46.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.995285 29333 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:45:46.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.995697 29333 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:45:46.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.996016 29333 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:45:46.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.996335 29333 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:45:46.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.996811 29333 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:45:46.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:46.997303 29333 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37013/jwks.json
16:45:47.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.019731 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } attempt: 7
16:45:47.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.052273 29256 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } has no permanent_uuid. Determining permanent_uuid...
16:45:47.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:47.070676 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:47.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.073770 29333 master_runner.cc:386] Master server non-default flags:
16:45:47.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:47.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:47.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:47.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:45:47.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/data
16:45:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/wal
16:45:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:45:47.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:45:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.60:45375
16:45:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:45:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:45:47.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:45:47.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37013/jwks.json
16:45:47.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:47.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:47.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/data/info.pb
16:45:47.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.60
16:45:47.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:47.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:45:47.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:47.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:45:47.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:45:47.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:47.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/logs
16:45:47.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:45:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:45:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:45:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:45:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:45:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:45:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:45:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:45:47.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:45:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.076122 29333 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:45:47.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.079841 29333 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:45:47.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:47.116652 29345 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:47.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.129626 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 1
16:45:47.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:47.141140 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:47.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.223191 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 2
16:45:47.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:47.238451 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:47.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.250887 29331 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 5
16:45:47.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:47.267731 29331 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:47.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.325678 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 3
16:45:47.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:47.337414 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:47.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.478359 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 4
16:45:47.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:47.488948 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:47.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.772822 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 5
16:45:47.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:47.785511 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:47.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:47.815759 29331 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 6
16:45:47.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:47.829783 29331 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:48.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.338443 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 6
16:45:48.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:48.352242 29256 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:48.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:48.520200 29344 debug-util.cc:398] Leaking SignalData structure 0x7b0800037520 after lost signal to thread 29333
16:45:48.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:48.663216 29344 kernel_stack_watchdog.cc:198] Thread 29333 stuck at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:642 for 390ms:
16:45:48.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:45:48.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:45:48.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:45:48.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:45:48.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:48.663749 29333 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.545s user 0.539s sys 1.003s
16:45:48.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:47.120692 29346 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:48.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:48.664405 29333 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.546s user 0.539s sys 1.003s
16:45:48.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:48.665438 29347 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:45:48.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.666531 29333 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:45:48.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:48.666853 29348 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:48.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.669849 29333 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:45:48.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.674345 29333 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:45:48.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.675702 29333 hybrid_clock.cc:648] HybridClock initialized: now 1761756348675665 us; error 45 us; skew 500 ppm
16:45:48.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.676611 29333 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:45:48.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.691056 29333 webserver.cc:492] Webserver started at http://127.28.126.60:45599/ using document root <none> and password file <none>
16:45:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.692381 29333 fs_manager.cc:362] Metadata directory not provided
16:45:48.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.692585 29333 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:45:48.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.693022 29333 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:45:48.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.699306 29333 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/data/instance:
16:45:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54749920d81a4ccfab7b7f4e9754b9dd"
16:45:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:48 on dist-test-slave-wkbt"
16:45:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.700491 29333 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/wal/instance:
16:45:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54749920d81a4ccfab7b7f4e9754b9dd"
16:45:48.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:48 on dist-test-slave-wkbt"
16:45:48.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.709923 29333 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.000s sys 0.008s
16:45:48.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.722225 29359 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:45:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.723955 29333 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.005s sys 0.001s
16:45:48.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.724368 29333 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/wal
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "54749920d81a4ccfab7b7f4e9754b9dd"
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:48 on dist-test-slave-wkbt"
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.724869 29333 fs_report.cc:389] FS layout report
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/wal
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/wal
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/data/data
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:45:48.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:45:48.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:45:48.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:45:48.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:45:48.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:45:48.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:45:48.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.804804 29333 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:45:48.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.808333 29333 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:45:48.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.809055 29333 kserver.cc:163] Server-wide thread pool size limit: 3276
16:45:48.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:48.890266 29331 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 7
16:45:48.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:48.905632 29331 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.28.126.60:45375: Network error: Client connection negotiation failed: client connection to 127.28.126.60:45375: connect: Connection refused (error 111)
16:45:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.052742 29333 rpc_server.cc:307] RPC server started. Bound to: 127.28.126.60:45375
16:45:49.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.053028 29412 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.126.60:45375 every 8 connection(s)
16:45:49.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.056104 29333 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/data/info.pb
16:45:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.064517 29176 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 29333
16:45:49.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.065038 29176 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/master-2/wal/instance
16:45:49.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.072243 29413 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:45:49.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.102661 29413 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } has no permanent_uuid. Determining permanent_uuid...
16:45:49.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.145481 29413 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } has no permanent_uuid. Determining permanent_uuid...
16:45:49.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.162130 29413 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } has no permanent_uuid. Determining permanent_uuid...
16:45:49.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.189976 29413 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd: Bootstrap starting.
16:45:49.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.201462 29413 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd: Neither blocks nor log segments found. Creating new log.
16:45:49.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.206492 29413 log.cc:826] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd: Log is configured to *not* fsync() on all Append() calls
16:45:49.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.217108 29413 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd: No bootstrap required, opened a new log
16:45:49.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.250345 29413 raft_consensus.cc:359] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } }
16:45:49.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.251398 29413 raft_consensus.cc:385] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:45:49.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.251811 29413 raft_consensus.cc:740] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 54749920d81a4ccfab7b7f4e9754b9dd, State: Initialized, Role: FOLLOWER
16:45:49.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.253024 29413 consensus_queue.cc:260] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } }
16:45:49.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.257453 29422 sys_catalog.cc:455] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } } }
16:45:49.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.258531 29422 sys_catalog.cc:458] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [sys.catalog]: This master's current role is: FOLLOWER
16:45:49.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.260367 29413 sys_catalog.cc:565] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [sys.catalog]: configured and running, proceeding with master startup.
16:45:49.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:49.321318 29365 tablet.cc:2378] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:45:49.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:49.340312 29433 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:45:49.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:49.341260 29433 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:45:49.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.320704 29333 master_runner.cc:186] Error getting master registration for 127.28.126.62:44765: OK, instance_id { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" instance_seqno: 1761756344634026 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:45:49.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.353266 29333 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:45:49.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.385811 29256 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 7
16:45:49.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.419440 29256 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d: Bootstrap starting.
16:45:49.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.424677 29422 raft_consensus.cc:493] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:45:49.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.425432 29422 raft_consensus.cc:515] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } }
16:45:49.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.431025 29422 leader_election.cc:290] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e3c1e3bcc76842f997ef6a4789353b8d (127.28.126.62:44765), 4df8813fc42646d9bb9369ea803d66d3 (127.28.126.61:45025)
16:45:49.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.436407 29256 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d: Neither blocks nor log segments found. Creating new log.
16:45:49.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.442723 29256 log.cc:826] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d: Log is configured to *not* fsync() on all Append() calls
16:45:49.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.455837 29256 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d: No bootstrap required, opened a new log
16:45:49.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.489682 29256 raft_consensus.cc:359] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } }
16:45:49.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.491518 29256 raft_consensus.cc:385] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:45:49.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.492203 29256 raft_consensus.cc:740] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e3c1e3bcc76842f997ef6a4789353b8d, State: Initialized, Role: FOLLOWER
16:45:49.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.494035 29256 consensus_queue.cc:260] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } }
16:45:49.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.534021 29256 sys_catalog.cc:565] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [sys.catalog]: configured and running, proceeding with master startup.
16:45:49.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.530546 29437 sys_catalog.cc:455] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } } }
16:45:49.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.535960 29437 sys_catalog.cc:458] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [sys.catalog]: This master's current role is: FOLLOWER
16:45:49.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.546743 29231 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" is_pre_election: true
16:45:49.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.550416 29231 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 54749920d81a4ccfab7b7f4e9754b9dd in term 0.
16:45:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.555631 29360 leader_election.cc:304] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 54749920d81a4ccfab7b7f4e9754b9dd, e3c1e3bcc76842f997ef6a4789353b8d; no voters:
16:45:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.557405 29422 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [term 0 FOLLOWER]: Leader pre-election won for term 1
16:45:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.557989 29422 raft_consensus.cc:493] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:45:49.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.558571 29422 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [term 0 FOLLOWER]: Advancing to term 1
16:45:49.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:49.595526 29210 tablet.cc:2378] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:45:49.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.604984 29422 raft_consensus.cc:515] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } }
16:45:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.623363 29231 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e3c1e3bcc76842f997ef6a4789353b8d"
16:45:49.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.624002 29231 raft_consensus.cc:3060] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [term 0 FOLLOWER]: Advancing to term 1
16:45:49.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.637233 29422 leader_election.cc:290] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [CANDIDATE]: Term 1 election: Requested vote from peers e3c1e3bcc76842f997ef6a4789353b8d (127.28.126.62:44765), 4df8813fc42646d9bb9369ea803d66d3 (127.28.126.61:45025)
16:45:49.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.638751 29231 raft_consensus.cc:2468] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 54749920d81a4ccfab7b7f4e9754b9dd in term 1.
16:45:49.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.640908 29360 leader_election.cc:304] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 54749920d81a4ccfab7b7f4e9754b9dd, e3c1e3bcc76842f997ef6a4789353b8d; no voters:
16:45:49.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.642071 29422 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [term 1 FOLLOWER]: Leader election won for term 1
16:45:49.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.664427 29422 raft_consensus.cc:697] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [term 1 LEADER]: Becoming Leader. State: Replica: 54749920d81a4ccfab7b7f4e9754b9dd, State: Running, Role: LEADER
16:45:49.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.669636 29422 consensus_queue.cc:237] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } }
16:45:49.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.692404 29447 sys_catalog.cc:455] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [sys.catalog]: SysCatalogTable state changed. Reason: New leader 54749920d81a4ccfab7b7f4e9754b9dd. Latest consensus state: current_term: 1 leader_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } } }
16:45:49.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.694000 29447 sys_catalog.cc:458] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [sys.catalog]: This master's current role is: LEADER
16:45:49.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:49.706279 29449 catalog_manager.cc:1568] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:45:49.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:49.707108 29449 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:45:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.723418 29452 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
16:45:49.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.733660 29452 catalog_manager.cc:1494] Initializing Kudu cluster ID...
16:45:49.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.686693 29188 master_runner.cc:186] Error getting master registration for 127.28.126.61:45025: OK, instance_id { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" instance_seqno: 1761756346471396 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:45:49.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.752210 29188 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:45:49.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.768537 29231 raft_consensus.cc:1275] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [term 1 FOLLOWER]: Refusing update from remote peer 54749920d81a4ccfab7b7f4e9754b9dd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:45:49.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.771816 29447 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [LEADER]: Connected to new peer: Peer: permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
16:45:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.815546 29437 sys_catalog.cc:455] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 54749920d81a4ccfab7b7f4e9754b9dd. Latest consensus state: current_term: 1 leader_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } } }
16:45:49.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.816356 29437 sys_catalog.cc:458] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [sys.catalog]: This master's current role is: FOLLOWER
16:45:49.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.821513 29453 mvcc.cc:204] Tried to move back new op lower bound from 7216154008621617152 to 7216154008267898880. Current Snapshot: MvccSnapshot[applied={T|T < 7216154008621617152}]
16:45:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.827114 29452 catalog_manager.cc:1357] Generated new cluster ID: 0cfa879cca3a413e99bf9a768eae6d82
16:45:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.827632 29452 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
16:45:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.827976 29422 sys_catalog.cc:455] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } } }
16:45:49.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.828718 29422 sys_catalog.cc:458] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [sys.catalog]: This master's current role is: LEADER
16:45:49.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.850144 29437 sys_catalog.cc:455] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } } }
16:45:49.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.850872 29437 sys_catalog.cc:458] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d [sys.catalog]: This master's current role is: FOLLOWER
16:45:49.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.872586 29452 catalog_manager.cc:1380] Generated new certificate authority record
16:45:49.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.876133 29452 catalog_manager.cc:1514] Loading token signing keys...
16:45:49.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.911386 29452 catalog_manager.cc:6022] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd: Generated new TSK 0
16:45:49.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.912554 29452 catalog_manager.cc:1524] Initializing in-progress tserver states...
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:49.965461 29176 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/wal
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/data
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/logs
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/data/info.pb
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:49.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:45:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:45:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:45:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:45:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:45:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:45:49.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.1:0
16:45:49.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.126.1
16:45:49.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.1
16:45:49.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:49.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:49.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:49.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:49.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:49.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:45:50.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:50.662194 29458 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:45:50.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:50.663332 29458 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:45:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:50.664368 29458 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:45:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:50.715737 29449 catalog_manager.cc:1269] Loaded cluster ID: 0cfa879cca3a413e99bf9a768eae6d82
16:45:50.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:50.716498 29449 catalog_manager.cc:1562] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d: loading cluster ID for follower catalog manager: success
16:45:50.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:50.725152 29458 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:45:50.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:50.726939 29458 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.126.1
16:45:50.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:50.737731 29449 catalog_manager.cc:1584] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d: acquiring CA information for follower catalog manager: success
16:45:50.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:50.750128 29449 catalog_manager.cc:1612] T 00000000000000000000000000000000 P e3c1e3bcc76842f997ef6a4789353b8d: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:45:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:50.792989 29458 tablet_server_runner.cc:78] Tablet server non-default flags:
16:45:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:45:50.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/data
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/wal
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.1:0
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/data/info.pb
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.1
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.126.1
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/logs
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:45:50.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:45:50.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:45:50.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:45:50.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:45:50.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:45:50.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:45:50.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:45:50.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:50.796824 29458 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:45:50.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:50.799454 29458 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:45:50.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:50.835755 29464 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:50.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:50.996320 29331 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } attempt: 8
16:45:51.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:51.212543 29363 proxy.cc:239] Call had error, refreshing address and retrying: Timed out: RequestConsensusVote RPC to 127.28.126.61:45025 timed out after 1.575s (SENT)
16:45:51.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.301384 29331 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3: Bootstrap starting.
16:45:51.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.535063 29331 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3: Neither blocks nor log segments found. Creating new log.
16:45:51.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.632050 29331 log.cc:826] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3: Log is configured to *not* fsync() on all Append() calls
16:45:51.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.662385 29331 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3: No bootstrap required, opened a new log
16:45:51.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.775997 29331 raft_consensus.cc:359] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } }
16:45:51.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.777320 29331 raft_consensus.cc:385] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:45:51.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.777840 29331 raft_consensus.cc:740] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4df8813fc42646d9bb9369ea803d66d3, State: Initialized, Role: FOLLOWER
16:45:51.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.779047 29331 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } }
16:45:51.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.791018 29476 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } } }
16:45:51.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.801978 29476 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [sys.catalog]: This master's current role is: FOLLOWER
16:45:51.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.802030 29303 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4df8813fc42646d9bb9369ea803d66d3"
16:45:51.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.796586 29302 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4df8813fc42646d9bb9369ea803d66d3" is_pre_election: true
16:45:51.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.797681 29306 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4df8813fc42646d9bb9369ea803d66d3" is_pre_election: true
16:45:51.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:51.806821 29306 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:51230 (request call id 1) took 2376 ms (client timeout 2228 ms). Trace:
16:45:51.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:51.807530 29306 rpcz_store.cc:269] 1029 16:45:49.430325 (+ 0us) service_pool.cc:168] Inserting onto call queue
16:45:51.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:45:49.430516 (+ 191us) service_pool.cc:225] Handling call
16:45:51.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:45:51.806703 (+2376187us) inbound_call.cc:173] Queueing success response
16:45:51.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
16:45:51.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.799249 29304 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [term 0 FOLLOWER]: Advancing to term 1
16:45:51.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.800652 29305 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4df8813fc42646d9bb9369ea803d66d3"
16:45:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.796267 29331 sys_catalog.cc:565] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [sys.catalog]: configured and running, proceeding with master startup.
16:45:51.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:51.816319 29305 rpcz_store.cc:267] Call kudu.consensus.ConsensusService.RequestConsensusVote from 127.0.0.1:51230 (request call id 2) took 2179 ms (client timeout 1575 ms). Trace:
16:45:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:51.817389 29305 rpcz_store.cc:269] 1029 16:45:49.636950 (+ 0us) service_pool.cc:168] Inserting onto call queue
16:45:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:45:49.637135 (+ 185us) service_pool.cc:225] Handling call
16:45:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:45:51.816169 (+2179034us) inbound_call.cc:173] Queueing success response
16:45:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
16:45:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.836006 29304 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [term 1 FOLLOWER]: Refusing update from remote peer 54749920d81a4ccfab7b7f4e9754b9dd: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:45:51.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.838372 29304 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.UpdateConsensus from 127.0.0.1:51230 (request call id 3) took 2072 ms. Trace:
16:45:51.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.839072 29304 rpcz_store.cc:276] 1029 16:45:49.766503 (+ 0us) service_pool.cc:168] Inserting onto call queue
16:45:51.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:45:49.766741 (+ 238us) service_pool.cc:225] Handling call
16:45:51.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:45:51.798748 (+2032007us) raft_consensus.cc:1471] Updating replica for 0 ops
16:45:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:45:51.837548 (+ 38800us) raft_consensus.cc:1697] Filling consensus response to leader.
16:45:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:45:51.838257 (+ 709us) inbound_call.cc:173] Queueing success response
16:45:51.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {}
16:45:51.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.842147 29473 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [LEADER]: Connected to new peer: Peer: permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
16:45:51.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.982538 29476 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 54749920d81a4ccfab7b7f4e9754b9dd. Latest consensus state: current_term: 1 leader_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } } }
16:45:51.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.983361 29476 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [sys.catalog]: This master's current role is: FOLLOWER
16:45:51.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.982805 29485 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } } }
16:45:51.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:51.983798 29485 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3 [sys.catalog]: This master's current role is: FOLLOWER
16:45:52.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:52.032276 29493 catalog_manager.cc:1269] Loaded cluster ID: 0cfa879cca3a413e99bf9a768eae6d82
16:45:52.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:52.032580 29493 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3: loading cluster ID for follower catalog manager: success
16:45:52.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:52.052484 29493 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3: acquiring CA information for follower catalog manager: success
16:45:52.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:52.058231 29493 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 4df8813fc42646d9bb9369ea803d66d3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:45:52.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:52.392923 29496 sys_catalog.cc:455] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e3c1e3bcc76842f997ef6a4789353b8d" member_type: VOTER last_known_addr { host: "127.28.126.62" port: 44765 } } peers { permanent_uuid: "4df8813fc42646d9bb9369ea803d66d3" member_type: VOTER last_known_addr { host: "127.28.126.61" port: 45025 } } peers { permanent_uuid: "54749920d81a4ccfab7b7f4e9754b9dd" member_type: VOTER last_known_addr { host: "127.28.126.60" port: 45375 } } }
16:45:52.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:52.394718 29496 sys_catalog.cc:458] T 00000000000000000000000000000000 P 54749920d81a4ccfab7b7f4e9754b9dd [sys.catalog]: This master's current role is: LEADER
16:45:52.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:52.228758 29463 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 29458
16:45:52.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:52.963506 29466 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2116 milliseconds
16:45:52.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:52.967742 29468 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:52.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:52.964063 29458 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.114s user 0.392s sys 0.758s
16:45:52.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:52.968717 29458 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.119s user 0.392s sys 0.758s
16:45:52.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:52.969007 29458 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:45:52.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:50.855394 29465 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:52.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:52.974025 29458 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:45:52.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:52.987082 29458 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:45:52.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:52.988942 29458 hybrid_clock.cc:648] HybridClock initialized: now 1761756352988901 us; error 62 us; skew 500 ppm
16:45:52.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:52.990343 29458 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:45:53.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.010361 29458 webserver.cc:492] Webserver started at http://127.28.126.1:39641/ using document root <none> and password file <none>
16:45:53.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.012059 29458 fs_manager.cc:362] Metadata directory not provided
16:45:53.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.012549 29458 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:45:53.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.013258 29458 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:45:53.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.020025 29458 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/data/instance:
16:45:53.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3282d1f0a0f14bf68633a09e41b90014"
16:45:53.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:53 on dist-test-slave-wkbt"
16:45:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.021929 29458 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/wal/instance:
16:45:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3282d1f0a0f14bf68633a09e41b90014"
16:45:53.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:53 on dist-test-slave-wkbt"
16:45:53.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.037665 29458 fs_manager.cc:696] Time spent creating directory manager: real 0.015s user 0.011s sys 0.002s
16:45:53.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.051759 29503 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:45:53.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.053649 29458 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.007s sys 0.005s
16:45:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.054345 29458 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/wal
16:45:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3282d1f0a0f14bf68633a09e41b90014"
16:45:53.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:53 on dist-test-slave-wkbt"
16:45:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.055042 29458 fs_report.cc:389] FS layout report
16:45:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:45:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/wal
16:45:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/wal
16:45:53.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/data/data
16:45:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:45:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:45:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:45:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:45:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:45:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:45:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:45:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:45:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:45:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:45:53.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:45:53.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.171242 29458 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:45:53.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.173820 29458 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:45:53.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.174729 29458 kserver.cc:163] Server-wide thread pool size limit: 3276
16:45:53.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.184509 29458 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:45:53.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.190584 29458 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:45:53.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.190879 29458 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:45:53.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.191215 29458 ts_tablet_manager.cc:616] Registered 0 tablets
16:45:53.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.191448 29458 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:45:53.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.567255 29458 rpc_server.cc:307] RPC server started. Bound to: 127.28.126.1:43079
16:45:53.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.567404 29615 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.126.1:43079 every 8 connection(s)
16:45:53.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.570307 29458 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/data/info.pb
16:45:53.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.574038 29176 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 29458
16:45:53.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.575416 29176 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-0/wal/instance
16:45:53.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.589030 29176 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:53.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:53.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/wal
16:45:53.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/data
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/logs
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/data/info.pb
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:45:53.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:45:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:45:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.2:0
16:45:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.126.2
16:45:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.2
16:45:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:53.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:53.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:53.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:53.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:45:53.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.651367 29616 heartbeater.cc:344] Connected to a master server at 127.28.126.61:45025
16:45:53.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.651844 29616 heartbeater.cc:461] Registering TS with master...
16:45:53.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.653254 29616 heartbeater.cc:507] Master 127.28.126.61:45025 requested a full tablet report, sending...
16:45:53.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.656955 29296 ts_manager.cc:194] Registered new tserver with Master: 3282d1f0a0f14bf68633a09e41b90014 (127.28.126.1:43079)
16:45:53.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.686792 29617 heartbeater.cc:344] Connected to a master server at 127.28.126.60:45375
16:45:53.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.687273 29617 heartbeater.cc:461] Registering TS with master...
16:45:53.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.688202 29617 heartbeater.cc:507] Master 127.28.126.60:45375 requested a full tablet report, sending...
16:45:53.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.691000 29378 ts_manager.cc:194] Registered new tserver with Master: 3282d1f0a0f14bf68633a09e41b90014 (127.28.126.1:43079)
16:45:53.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.693750 29378 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.126.1:39459
16:45:53.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.701239 29621 heartbeater.cc:344] Connected to a master server at 127.28.126.62:44765
16:45:53.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.702015 29621 heartbeater.cc:461] Registering TS with master...
16:45:53.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.702943 29621 heartbeater.cc:507] Master 127.28.126.62:44765 requested a full tablet report, sending...
16:45:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:53.706314 29220 ts_manager.cc:194] Registered new tserver with Master: 3282d1f0a0f14bf68633a09e41b90014 (127.28.126.1:43079)
16:45:54.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:54.081336 29618 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:45:54.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:54.082023 29618 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:45:54.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:54.082743 29618 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:45:54.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:54.116618 29618 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:45:54.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:54.117605 29618 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.126.2
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:54.154551 29618 tablet_server_runner.cc:78] Tablet server non-default flags:
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/data
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/wal
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.2:0
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/data/info.pb
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.2
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:54.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:54.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:45:54.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:54.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:45:54.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:45:54.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:54.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.126.2
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/logs
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:45:54.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:54.156126 29618 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:45:54.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:54.157794 29618 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:45:54.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:54.171321 29632 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:54.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:54.698884 29617 heartbeater.cc:499] Master 127.28.126.60:45375 was elected leader, sending a full tablet report...
16:45:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:55.573622 29631 debug-util.cc:398] Leaking SignalData structure 0x7b0800037500 after lost signal to thread 29618
16:45:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:54.174235 29633 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:55.638741 29618 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.465s user 0.520s sys 0.926s
16:45:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:55.639065 29618 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.466s user 0.520s sys 0.926s
16:45:55.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:55.641114 29635 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:55.643286 29634 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
16:45:55.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.643309 29618 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:45:55.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.644483 29618 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:45:55.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.646482 29618 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:45:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.647799 29618 hybrid_clock.cc:648] HybridClock initialized: now 1761756355647759 us; error 59 us; skew 500 ppm
16:45:55.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.648576 29618 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:45:55.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.655493 29618 webserver.cc:492] Webserver started at http://127.28.126.2:44987/ using document root <none> and password file <none>
16:45:55.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.656728 29618 fs_manager.cc:362] Metadata directory not provided
16:45:55.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.657011 29618 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:45:55.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.657553 29618 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:45:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.663676 29618 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/data/instance:
16:45:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4067c4af16a247878e0cced804616ff6"
16:45:55.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:55 on dist-test-slave-wkbt"
16:45:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.665226 29618 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/wal/instance:
16:45:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4067c4af16a247878e0cced804616ff6"
16:45:55.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:55 on dist-test-slave-wkbt"
16:45:55.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.674522 29618 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.005s
16:45:55.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.681692 29643 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:45:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.682967 29618 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.005s
16:45:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.683365 29618 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/wal
16:45:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4067c4af16a247878e0cced804616ff6"
16:45:55.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:55 on dist-test-slave-wkbt"
16:45:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.683784 29618 fs_report.cc:389] FS layout report
16:45:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:45:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/wal
16:45:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/wal
16:45:55.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/data/data
16:45:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:45:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:45:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:45:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:45:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:45:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:45:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:45:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:45:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:45:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:45:55.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:45:55.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.748180 29618 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:45:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.749786 29618 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:45:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.750249 29618 kserver.cc:163] Server-wide thread pool size limit: 3276
16:45:55.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.753007 29618 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:45:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.757258 29618 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:45:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.757457 29618 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:45:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.757645 29618 ts_tablet_manager.cc:616] Registered 0 tablets
16:45:55.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.757778 29618 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:45:55.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.929605 29618 rpc_server.cc:307] RPC server started. Bound to: 127.28.126.2:42399
16:45:55.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.929716 29755 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.126.2:42399 every 8 connection(s)
16:45:55.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.932097 29618 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/data/info.pb
16:45:55.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.936834 29176 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 29618
16:45:55.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.937325 29176 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-1/wal/instance
16:45:55.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.958456 29176 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/wal
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/data
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/logs
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/data/info.pb
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.3:0
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.126.3
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.3
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
16:45:55.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.986003 29756 heartbeater.cc:344] Connected to a master server at 127.28.126.61:45025
16:45:55.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.987313 29756 heartbeater.cc:461] Registering TS with master...
16:45:55.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.989216 29756 heartbeater.cc:507] Master 127.28.126.61:45025 requested a full tablet report, sending...
16:45:55.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:55.994139 29296 ts_manager.cc:194] Registered new tserver with Master: 4067c4af16a247878e0cced804616ff6 (127.28.126.2:42399)
16:45:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.006729 29757 heartbeater.cc:344] Connected to a master server at 127.28.126.60:45375
16:45:56.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.007175 29757 heartbeater.cc:461] Registering TS with master...
16:45:56.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.008078 29757 heartbeater.cc:507] Master 127.28.126.60:45375 requested a full tablet report, sending...
16:45:56.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.010128 29760 heartbeater.cc:344] Connected to a master server at 127.28.126.62:44765
16:45:56.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.010325 29378 ts_manager.cc:194] Registered new tserver with Master: 4067c4af16a247878e0cced804616ff6 (127.28.126.2:42399)
16:45:56.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.010506 29760 heartbeater.cc:461] Registering TS with master...
16:45:56.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.011286 29760 heartbeater.cc:507] Master 127.28.126.62:44765 requested a full tablet report, sending...
16:45:56.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.012066 29378 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.126.2:33063
16:45:56.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.013751 29220 ts_manager.cc:194] Registered new tserver with Master: 4067c4af16a247878e0cced804616ff6 (127.28.126.2:42399)
16:45:56.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:56.320472 29763 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:45:56.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:56.320976 29763 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:45:56.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:56.321413 29763 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:45:56.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:56.353648 29763 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:45:56.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:56.354950 29763 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.28.126.3
16:45:56.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.392539 29763 tablet_server_runner.cc:78] Tablet server non-default flags:
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.28.126.20:34663
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/data
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/wal
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.28.126.3:0
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/data/info.pb
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.28.126.3
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.28.126.62:44765,127.28.126.61:45025,127.28.126.60:45375
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.28.126.3
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/logs
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:45:56.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:45:56.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:45:56.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:45:56.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:45:56.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:45:56.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:45:56.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:45:56.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.394213 29763 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:45:56.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.395886 29763 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:45:56.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:56.409214 29772 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:57.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.016659 29757 heartbeater.cc:499] Master 127.28.126.60:45375 was elected leader, sending a full tablet report...
16:45:57.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:56.413362 29775 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:57.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:56.414113 29763 server_base.cc:1047] running on GCE node
16:45:57.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:45:56.412050 29773 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:45:57.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.601402 29763 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:45:57.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.604336 29763 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:45:57.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.605794 29763 hybrid_clock.cc:648] HybridClock initialized: now 1761756357605745 us; error 66 us; skew 500 ppm
16:45:57.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.606609 29763 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:45:57.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.612905 29763 webserver.cc:492] Webserver started at http://127.28.126.3:46139/ using document root <none> and password file <none>
16:45:57.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.613904 29763 fs_manager.cc:362] Metadata directory not provided
16:45:57.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.614089 29763 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:45:57.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.614485 29763 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:45:57.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.618964 29763 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/data/instance:
16:45:57.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9f2ed8d6a67470cb6560102d23abf8f"
16:45:57.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:57 on dist-test-slave-wkbt"
16:45:57.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.620057 29763 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/wal/instance:
16:45:57.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9f2ed8d6a67470cb6560102d23abf8f"
16:45:57.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:57 on dist-test-slave-wkbt"
16:45:57.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.627125 29763 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
16:45:57.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.632791 29783 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:45:57.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.633920 29763 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
16:45:57.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.634209 29763 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/wal
16:45:57.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b9f2ed8d6a67470cb6560102d23abf8f"
16:45:57.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:45:57 on dist-test-slave-wkbt"
16:45:57.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.634505 29763 fs_report.cc:389] FS layout report
16:45:57.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:45:57.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/wal
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/wal
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/data/data
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:45:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:45:57.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.688215 29763 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:45:57.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.689704 29763 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:45:57.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.690094 29763 kserver.cc:163] Server-wide thread pool size limit: 3276
16:45:57.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.692656 29763 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:45:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.697088 29763 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:45:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.697284 29763 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:45:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.697474 29763 ts_tablet_manager.cc:616] Registered 0 tablets
16:45:57.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.697602 29763 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:45:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.862488 29763 rpc_server.cc:307] RPC server started. Bound to: 127.28.126.3:45327
16:45:57.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.862625 29896 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.28.126.3:45327 every 8 connection(s)
16:45:57.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.865510 29763 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/data/info.pb
16:45:57.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.870323 29176 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 29763
16:45:57.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.870803 29176 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster4665255028904010197/ts-2/wal/instance
16:45:57.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.926400 29898 heartbeater.cc:344] Connected to a master server at 127.28.126.60:45375
16:45:57.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.926893 29898 heartbeater.cc:461] Registering TS with master...
16:45:57.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.928948 29898 heartbeater.cc:507] Master 127.28.126.60:45375 requested a full tablet report, sending...
16:45:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.930397 29899 heartbeater.cc:344] Connected to a master server at 127.28.126.62:44765
16:45:57.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.930708 29899 heartbeater.cc:461] Registering TS with master...
16:45:57.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.931686 29899 heartbeater.cc:507] Master 127.28.126.62:44765 requested a full tablet report, sending...
16:45:57.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.933337 29378 ts_manager.cc:194] Registered new tserver with Master: b9f2ed8d6a67470cb6560102d23abf8f (127.28.126.3:45327)
16:45:57.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.934924 29378 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.28.126.3:42381
16:45:57.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.938185 29220 ts_manager.cc:194] Registered new tserver with Master: b9f2ed8d6a67470cb6560102d23abf8f (127.28.126.3:45327)
16:45:57.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.939581 29897 heartbeater.cc:344] Connected to a master server at 127.28.126.61:45025
16:45:57.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.939931 29897 heartbeater.cc:461] Registering TS with master...
16:45:57.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.940867 29897 heartbeater.cc:507] Master 127.28.126.61:45025 requested a full tablet report, sending...
16:45:57.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.943568 29296 ts_manager.cc:194] Registered new tserver with Master: b9f2ed8d6a67470cb6560102d23abf8f (127.28.126.3:45327)
16:45:57.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:57.948644 29176 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
16:45:57.960 [DEBUG - main] (MiniKuduCluster.java:195) Response:
16:45:57.963 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
16:45:57.977 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.28.126.62"
port: 44765
}
bound_http_address {
host: "127.28.126.62"
port: 42703
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.28.126.61"
port: 45025
}
bound_http_address {
host: "127.28.126.61"
port: 40893
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.28.126.60"
port: 45375
}
bound_http_address {
host: "127.28.126.60"
port: 45599
}
}
}
16:45:57.982 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:45:57.985 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.28.126.1"
port: 43079
}
bound_http_address {
host: "127.28.126.1"
port: 39641
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.28.126.2"
port: 42399
}
bound_http_address {
host: "127.28.126.2"
port: 44987
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.28.126.3"
port: 45327
}
bound_http_address {
host: "127.28.126.3"
port: 46139
}
}
}
16:45:57.985 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
16:45:58.186 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
16:45:58.211 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
16:45:58.330 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskXsbxVh/test-tmp
16:45:58.330 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
16:45:58.331 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskXsbxVh/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
16:45:58.334 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
16:45:58.720 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:45:58.721 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
16:45:58.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:58.724694 29908 mini_oidc.cc:150] Starting JWKS server
16:45:58.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:58.728637 29908 webserver.cc:492] Webserver started at http://127.0.0.1:38345/ using document root <none> and password file <none>
16:45:58.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:58.729035 29908 mini_oidc.cc:186] Starting OIDC Discovery server
16:45:58.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:58.731226 29908 webserver.cc:492] Webserver started at http://127.0.0.1:34897/ using document root <none> and password file <none>
16:45:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
16:45:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/principal' for realm 'KRBTEST.COM',
16:45:58.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
16:45:58.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:45:58 dist-test-slave-wkbt krb5kdc[29920](info): setting up network...
16:45:58.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
16:45:58.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:45:58 dist-test-slave-wkbt krb5kdc[29920](info): set up 2 sockets
16:45:58.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:45:58 dist-test-slave-wkbt krb5kdc[29920](info): commencing operation
16:45:58.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
16:45:58.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:45:58.943804 29898 heartbeater.cc:499] Master 127.28.126.60:45375 was elected leader, sending a full tablet report...
16:46:00.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:00.838546 29908 mini_kdc.cc:121] Time spent starting KDC: real 2.065s user 0.006s sys 0.014s
16:46:00.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
16:46:00.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:46:00.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
16:46:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
16:46:00.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:46:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
16:46:00.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
16:46:00.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:46:00.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
16:46:00.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
16:46:00.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/test-user.keytab.
16:46:00.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/test-user.keytab.
16:46:00.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:00 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756360, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:00.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:46:00.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T16:46:00Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
16:46:00.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T16:46:00Z Disabled control of system clock
16:46:01.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.53.62@KRBTEST.COM; defaulting to no policy
16:46:01.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:01.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.53.62@KRBTEST.COM" created.
16:46:01.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:01.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:01.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:01.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.29.53.62@KRBTEST.COM; defaulting to no policy
16:46:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:01.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.29.53.62@KRBTEST.COM" created.
16:46:01.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:01.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:01.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:01.081586 29908 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/logs
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data/info.pb
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.62:41263
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.62
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.62
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:01.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:46:01.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:46:01.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.400942 29938 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:46:01.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.401499 29938 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:46:01.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.402133 29938 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:46:01.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.435616 29938 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:46:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.435868 29938 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:46:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.436141 29938 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:46:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.436398 29938 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:46:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.436604 29938 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:46:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.436795 29938 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:46:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.437005 29938 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38345/jwks.json
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:01.475545 29938 master_runner.cc:386] Master server non-default flags:
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.62
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.62:41263
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data/info.pb
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.62
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/logs
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:46:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:46:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:46:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:46:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:01.476935 29938 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:46:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:01.477979 29938 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:46:01.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.490291 29945 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:02.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:01.490299 29944 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:02.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:02.899122 29943 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 29938
16:46:02.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:02.932101 29938 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.442s user 0.388s sys 1.038s
16:46:02.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:02.933606 29938 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.444s user 0.388s sys 1.038s
16:46:02.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:02.934237 29947 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:02.939131 29946 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1448 milliseconds
16:46:02.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.939254 29938 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:46:02.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.940780 29938 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:46:02.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.943804 29938 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:46:02.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.945196 29938 hybrid_clock.cc:648] HybridClock initialized: now 1761756362945136 us; error 59 us; skew 500 ppm
16:46:02.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.945999 29938 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:46:02.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:02 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756362, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:02.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.952342 29938 init.cc:377] Logged in from keytab as kudu/127.29.53.62@KRBTEST.COM (short username kudu)
16:46:02.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.957129 29938 webserver.cc:492] Webserver started at http://127.29.53.62:40455/ using document root <none> and password file <none>
16:46:02.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.958022 29938 fs_manager.cc:362] Metadata directory not provided
16:46:02.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.958220 29938 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:46:02.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.958621 29938 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:46:02.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.962997 29938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data/instance:
16:46:02.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1869e0a1407440e79e0b04ecb71da9b0"
16:46:02.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:02 on dist-test-slave-wkbt"
16:46:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.964243 29938 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal/instance:
16:46:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1869e0a1407440e79e0b04ecb71da9b0"
16:46:02.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:02 on dist-test-slave-wkbt"
16:46:02.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.974077 29938 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.002s
16:46:02.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.980015 29957 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:46:02.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.981146 29938 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
16:46:02.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.981443 29938 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal
16:46:02.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1869e0a1407440e79e0b04ecb71da9b0"
16:46:02.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:02 on dist-test-slave-wkbt"
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:02.981751 29938 fs_report.cc:389] FS layout report
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data/data
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:46:02.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:46:03.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.042465 29938 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:46:03.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.056383 29938 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:46:03.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.056973 29938 kserver.cc:163] Server-wide thread pool size limit: 3276
16:46:03.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.144623 29938 rpc_server.cc:307] RPC server started. Bound to: 127.29.53.62:41263
16:46:03.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.144709 30009 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.53.62:41263 every 8 connection(s)
16:46:03.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.147347 29938 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data/info.pb
16:46:03.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.149953 29908 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 29938
16:46:03.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.150524 29908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal/instance
16:46:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.154947 30010 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:46:03.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.53.61@KRBTEST.COM; defaulting to no policy
16:46:03.171 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.168291 30010 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } has no permanent_uuid. Determining permanent_uuid...
16:46:03.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:03.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.53.61@KRBTEST.COM" created.
16:46:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:03 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756362, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.62@KRBTEST.COM for kudu/127.29.53.62@KRBTEST.COM
16:46:03.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:03.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:03.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:03.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.209507 30010 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } has no permanent_uuid. Determining permanent_uuid...
16:46:03.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.212231 29962 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.53.61:44169: connect: Connection refused (error 111)
16:46:03.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.29.53.61@KRBTEST.COM; defaulting to no policy
16:46:03.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:03.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.29.53.61@KRBTEST.COM" created.
16:46:03.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.216930 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.61:44169: Network error: Client connection negotiation failed: client connection to 127.29.53.61:44169: connect: Connection refused (error 111)
16:46:03.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:03.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:03.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.239290 29908 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal
16:46:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data
16:46:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:46:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:46:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:46:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/logs
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data/info.pb
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.61:44169
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.61
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.61
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:46:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:46:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:46:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:46:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:46:03.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.266445 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } attempt: 1
16:46:03.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.270284 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.61:44169: Network error: Client connection negotiation failed: client connection to 127.29.53.61:44169: connect: Connection refused (error 111)
16:46:03.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.338713 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } attempt: 2
16:46:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.343432 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.61:44169: Network error: Client connection negotiation failed: client connection to 127.29.53.61:44169: connect: Connection refused (error 111)
16:46:03.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.435063 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } attempt: 3
16:46:03.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.440307 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.61:44169: Network error: Client connection negotiation failed: client connection to 127.29.53.61:44169: connect: Connection refused (error 111)
16:46:03.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.583783 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } attempt: 4
16:46:03.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.588997 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.61:44169: Network error: Client connection negotiation failed: client connection to 127.29.53.61:44169: connect: Connection refused (error 111)
16:46:03.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.637264 30019 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:46:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.637805 30019 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:46:03.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.638622 30019 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:46:03.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.673518 30019 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:46:03.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.673815 30019 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:46:03.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.674227 30019 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:46:03.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.674539 30019 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:46:03.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.674835 30019 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:46:03.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.675163 30019 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:46:03.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.675400 30019 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38345/jwks.json
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.715234 30019 master_runner.cc:386] Master server non-default flags:
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.61
16:46:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.61:44169
16:46:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:46:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data/info.pb
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.61
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/logs
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.716569 30019 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:46:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.717576 30019 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:46:03.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.733379 30025 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:03.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:03.888566 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } attempt: 5
16:46:03.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.897377 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.61:44169: Network error: Client connection negotiation failed: client connection to 127.29.53.61:44169: connect: Connection refused (error 111)
16:46:04.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:04.445021 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } attempt: 6
16:46:04.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:04.456566 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.61:44169: Network error: Client connection negotiation failed: client connection to 127.29.53.61:44169: connect: Connection refused (error 111)
16:46:05.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:03.736500 30026 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:05.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.073825 30028 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:05.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.078322 30019 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.339s user 0.004s sys 0.006s
16:46:05.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.078678 30019 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.340s user 0.004s sys 0.006s
16:46:05.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.078905 30019 server_base.cc:1047] running on GCE node
16:46:05.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.080245 30019 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:46:05.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.083222 30019 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:46:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.084823 30019 hybrid_clock.cc:648] HybridClock initialized: now 1761756365084784 us; error 42 us; skew 500 ppm
16:46:05.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.085887 30019 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:46:05.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:05 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756365, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:05.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.094416 30019 init.cc:377] Logged in from keytab as kudu/127.29.53.61@KRBTEST.COM (short username kudu)
16:46:05.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.100298 30019 webserver.cc:492] Webserver started at http://127.29.53.61:35445/ using document root <none> and password file <none>
16:46:05.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.101193 30019 fs_manager.cc:362] Metadata directory not provided
16:46:05.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.101380 30019 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:46:05.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.101754 30019 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:46:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.106165 30019 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data/instance:
16:46:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "083204b027784cb697b1041fa88ef693"
16:46:05.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:05 on dist-test-slave-wkbt"
16:46:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.107376 30019 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal/instance:
16:46:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "083204b027784cb697b1041fa88ef693"
16:46:05.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:05 on dist-test-slave-wkbt"
16:46:05.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.115733 30019 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.001s
16:46:05.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.121632 30040 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:46:05.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.122690 30019 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.002s
16:46:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.123020 30019 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal
16:46:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "083204b027784cb697b1041fa88ef693"
16:46:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:05 on dist-test-slave-wkbt"
16:46:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.123394 30019 fs_report.cc:389] FS layout report
16:46:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:46:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal
16:46:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal
16:46:05.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data/data
16:46:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:46:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:46:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:46:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:46:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:46:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:46:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:46:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:46:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:46:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:46:05.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:46:05.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.201089 30019 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:46:05.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.207399 30019 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:46:05.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.208048 30019 kserver.cc:163] Server-wide thread pool size limit: 3276
16:46:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.364132 30019 rpc_server.cc:307] RPC server started. Bound to: 127.29.53.61:44169
16:46:05.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.364287 30092 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.53.61:44169 every 8 connection(s)
16:46:05.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.367121 30019 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data/info.pb
16:46:05.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.374627 29908 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 30019
16:46:05.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.375100 29908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal/instance
16:46:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.378588 30093 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:46:05.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.53.60@KRBTEST.COM; defaulting to no policy
16:46:05.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:05.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.53.60@KRBTEST.COM" created.
16:46:05.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.397473 30093 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } has no permanent_uuid. Determining permanent_uuid...
16:46:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:05.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:05.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:05 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756365, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.61@KRBTEST.COM for kudu/127.29.53.62@KRBTEST.COM
16:46:05.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.438570 30093 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } has no permanent_uuid. Determining permanent_uuid...
16:46:05.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.29.53.60@KRBTEST.COM; defaulting to no policy
16:46:05.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:05.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.29.53.60@KRBTEST.COM" created.
16:46:05.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:05 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756365, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.61@KRBTEST.COM for kudu/127.29.53.61@KRBTEST.COM
16:46:05.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:05.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:05.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.467810 29908 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/logs
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data/info.pb
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:46:05.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.60:38873
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.60
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.60
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:46:05.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:46:05.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.470504 30093 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } has no permanent_uuid. Determining permanent_uuid...
16:46:05.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.473007 30044 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:05.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.477982 30093 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:05.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.517131 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } attempt: 7
16:46:05.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.527587 30093 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 1
16:46:05.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.531533 30093 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:05.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:05 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756362, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.62@KRBTEST.COM for kudu/127.29.53.61@KRBTEST.COM
16:46:05.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.544106 30010 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } has no permanent_uuid. Determining permanent_uuid...
16:46:05.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.551465 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:05.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.599912 30093 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 2
16:46:05.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.604578 30093 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:05.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.609965 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 1
16:46:05.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.614470 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:05.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.696149 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 2
16:46:05.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.696149 30093 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 3
16:46:05.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.701817 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:05.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.702808 30093 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:05.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.787263 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 3
16:46:05.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.793272 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:05.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.846221 30093 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 4
16:46:05.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.850348 30093 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:05.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:05.933725 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 4
16:46:05.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.940958 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.953104 30102 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:46:05.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.953662 30102 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:46:05.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.954283 30102 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:46:05.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.990139 30102 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:46:05.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.990401 30102 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:46:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.990851 30102 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:46:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.991269 30102 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:46:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.991489 30102 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:46:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.991683 30102 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:46:05.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:05.991829 30102 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38345/jwks.json
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:06.032606 30102 master_runner.cc:386] Master server non-default flags:
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.60
16:46:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.60:38873
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data/info.pb
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.60
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/logs
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:46:06.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:46:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:46:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:46:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:06.033912 30102 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:46:06.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:06.035303 30102 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:46:06.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:06.063048 30111 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:06.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:06.149977 30093 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 5
16:46:06.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:06.159574 30093 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:06.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:06.224402 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 5
16:46:06.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:06.232926 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:06.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:06.707222 30093 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 6
16:46:06.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:06.738472 30093 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:06.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:06.785820 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 6
16:46:06.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:06.798342 30010 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:07.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:06.065017 30112 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:07.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:07.474468 30102 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.406s user 0.008s sys 0.004s
16:46:07.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:07.474858 30102 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.406s user 0.008s sys 0.004s
16:46:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:07.477728 30114 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:07.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.480742 30102 server_base.cc:1047] running on GCE node
16:46:07.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.482285 30102 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:46:07.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.485844 30102 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:46:07.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.487336 30102 hybrid_clock.cc:648] HybridClock initialized: now 1761756367487213 us; error 108 us; skew 500 ppm
16:46:07.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.488811 30102 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:46:07.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:07 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756367, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:07.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.498776 30102 init.cc:377] Logged in from keytab as kudu/127.29.53.60@KRBTEST.COM (short username kudu)
16:46:07.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.513724 30102 webserver.cc:492] Webserver started at http://127.29.53.60:46741/ using document root <none> and password file <none>
16:46:07.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.514681 30102 fs_manager.cc:362] Metadata directory not provided
16:46:07.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.514932 30102 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:46:07.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.515414 30102 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:46:07.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.520192 30102 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data/instance:
16:46:07.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b76f6b3da73b432fb8397d6dd53510c1"
16:46:07.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:07 on dist-test-slave-wkbt"
16:46:07.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.521329 30102 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal/instance:
16:46:07.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b76f6b3da73b432fb8397d6dd53510c1"
16:46:07.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:07 on dist-test-slave-wkbt"
16:46:07.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.531320 30102 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.006s sys 0.005s
16:46:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.539906 30128 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:46:07.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.541446 30102 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.001s sys 0.003s
16:46:07.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.541843 30102 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal
16:46:07.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b76f6b3da73b432fb8397d6dd53510c1"
16:46:07.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:07 on dist-test-slave-wkbt"
16:46:07.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.542271 30102 fs_report.cc:389] FS layout report
16:46:07.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:46:07.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data/data
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:46:07.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:46:07.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.616549 30102 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:46:07.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.622467 30102 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:46:07.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.622884 30102 kserver.cc:163] Server-wide thread pool size limit: 3276
16:46:07.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.759971 30102 rpc_server.cc:307] RPC server started. Bound to: 127.29.53.60:38873
16:46:07.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.760509 30179 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.53.60:38873 every 8 connection(s)
16:46:07.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.764276 30102 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data/info.pb
16:46:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.767993 29908 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 30102
16:46:07.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.768520 29908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal/instance
16:46:07.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.776160 30180 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
16:46:07.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.799044 30093 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 7
16:46:07.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.802047 30180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } has no permanent_uuid. Determining permanent_uuid...
16:46:07.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:07 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756360, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.53.62@KRBTEST.COM
16:46:07.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.831924 30010 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } attempt: 7
16:46:07.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:07 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756365, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.61@KRBTEST.COM for kudu/127.29.53.60@KRBTEST.COM
16:46:07.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:07 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756367, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.60@KRBTEST.COM for kudu/127.29.53.62@KRBTEST.COM
16:46:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.866254 30180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } has no permanent_uuid. Determining permanent_uuid...
16:46:07.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:07 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756360, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.53.61@KRBTEST.COM
16:46:07.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.875625 30093 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: Bootstrap starting.
16:46:07.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:07 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756362, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.62@KRBTEST.COM for kudu/127.29.53.60@KRBTEST.COM
16:46:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.892419 30093 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: Neither blocks nor log segments found. Creating new log.
16:46:07.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.898053 30093 log.cc:826] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: Log is configured to *not* fsync() on all Append() calls
16:46:07.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:07 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756360, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.29.53.60@KRBTEST.COM
16:46:07.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:07 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756367, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.60@KRBTEST.COM for kudu/127.29.53.61@KRBTEST.COM
16:46:07.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.909279 30010 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: Bootstrap starting.
16:46:07.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.924880 30180 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } has no permanent_uuid. Determining permanent_uuid...
16:46:07.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:07 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756367, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.60@KRBTEST.COM for kudu/127.29.53.60@KRBTEST.COM
16:46:07.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.945097 30010 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: Neither blocks nor log segments found. Creating new log.
16:46:07.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.946744 30093 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: No bootstrap required, opened a new log
16:46:07.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.947525 30010 log.cc:826] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: Log is configured to *not* fsync() on all Append() calls
16:46:07.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.965319 30010 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: No bootstrap required, opened a new log
16:46:07.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.970403 30180 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1: Bootstrap starting.
16:46:07.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.973893 30093 raft_consensus.cc:359] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.974820 30093 raft_consensus.cc:385] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:46:07.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.975133 30093 raft_consensus.cc:740] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 083204b027784cb697b1041fa88ef693, State: Initialized, Role: FOLLOWER
16:46:07.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.976089 30093 consensus_queue.cc:260] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:07.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.982151 30195 sys_catalog.cc:455] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:07.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.982998 30180 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1: Neither blocks nor log segments found. Creating new log.
16:46:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.984288 30093 sys_catalog.cc:565] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: configured and running, proceeding with master startup.
16:46:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.985133 30195 sys_catalog.cc:458] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: This master's current role is: FOLLOWER
16:46:07.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.985227 30180 log.cc:826] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1: Log is configured to *not* fsync() on all Append() calls
16:46:07.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.991755 30010 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:07.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.993521 30010 raft_consensus.cc:385] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:46:07.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.994098 30010 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1869e0a1407440e79e0b04ecb71da9b0, State: Initialized, Role: FOLLOWER
16:46:07.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:07.995579 30010 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:08.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.004717 30199 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:08.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.005915 30199 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: This master's current role is: FOLLOWER
16:46:08.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.007964 30010 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: configured and running, proceeding with master startup.
16:46:08.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.014300 30180 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1: No bootstrap required, opened a new log
16:46:08.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:08.070128 29964 tablet.cc:2378] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:46:08.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.179219 30180 raft_consensus.cc:359] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:08.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.182648 30180 raft_consensus.cc:385] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
16:46:08.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.183698 30180 raft_consensus.cc:740] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b76f6b3da73b432fb8397d6dd53510c1, State: Initialized, Role: FOLLOWER
16:46:08.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:08.221704 30046 tablet.cc:2378] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:46:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.225824 30180 consensus_queue.cc:260] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:08.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:08.234064 30215 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:46:08.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:08.241444 30215 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:46:08.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.299088 30222 raft_consensus.cc:493] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:46:08.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.301649 30222 raft_consensus.cc:515] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:08.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.319491 29985 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b76f6b3da73b432fb8397d6dd53510c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1869e0a1407440e79e0b04ecb71da9b0" is_pre_election: true
16:46:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.313140 30220 sys_catalog.cc:455] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:08.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.326159 30220 sys_catalog.cc:458] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: This master's current role is: FOLLOWER
16:46:08.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.330332 29985 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b76f6b3da73b432fb8397d6dd53510c1 in term 0.
16:46:08.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.334770 30180 sys_catalog.cc:565] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: configured and running, proceeding with master startup.
16:46:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.337915 30068 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b76f6b3da73b432fb8397d6dd53510c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "083204b027784cb697b1041fa88ef693" is_pre_election: true
16:46:08.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.347672 30130 leader_election.cc:304] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1869e0a1407440e79e0b04ecb71da9b0, b76f6b3da73b432fb8397d6dd53510c1; no voters:
16:46:08.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.351554 30220 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 0 FOLLOWER]: Leader pre-election won for term 1
16:46:08.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.353477 30220 raft_consensus.cc:493] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:46:08.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.337846 30222 leader_election.cc:290] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 1869e0a1407440e79e0b04ecb71da9b0 (127.29.53.62:41263), 083204b027784cb697b1041fa88ef693 (127.29.53.61:44169)
16:46:08.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.355319 30220 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 0 FOLLOWER]: Advancing to term 1
16:46:08.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:08.382364 30134 tablet.cc:2378] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1: Can't schedule compaction. Clean time has not been advanced past its initial value.
16:46:08.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.352362 30068 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b76f6b3da73b432fb8397d6dd53510c1 in term 0.
16:46:08.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.400038 30220 raft_consensus.cc:515] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:08.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.436369 29985 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b76f6b3da73b432fb8397d6dd53510c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "1869e0a1407440e79e0b04ecb71da9b0"
16:46:08.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.428551 30220 leader_election.cc:290] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [CANDIDATE]: Term 1 election: Requested vote from peers 1869e0a1407440e79e0b04ecb71da9b0 (127.29.53.62:41263), 083204b027784cb697b1041fa88ef693 (127.29.53.61:44169)
16:46:08.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.440532 29985 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 0 FOLLOWER]: Advancing to term 1
16:46:08.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.449426 30068 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b76f6b3da73b432fb8397d6dd53510c1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "083204b027784cb697b1041fa88ef693"
16:46:08.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.452378 30068 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 0 FOLLOWER]: Advancing to term 1
16:46:08.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:08.502379 30221 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:46:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:08.506284 30221 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:46:08.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.539494 29985 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b76f6b3da73b432fb8397d6dd53510c1 in term 1.
16:46:08.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.545766 30130 leader_election.cc:304] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 1869e0a1407440e79e0b04ecb71da9b0, b76f6b3da73b432fb8397d6dd53510c1; no voters:
16:46:08.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.551573 30220 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 1 FOLLOWER]: Leader election won for term 1
16:46:08.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.558182 30220 raft_consensus.cc:697] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 1 LEADER]: Becoming Leader. State: Replica: b76f6b3da73b432fb8397d6dd53510c1, State: Running, Role: LEADER
16:46:08.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.566802 30220 consensus_queue.cc:237] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:08.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.364336 29938 master_runner.cc:186] Error getting master registration for 127.29.53.60:38873: OK, instance_id { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" instance_seqno: 1761756367719910 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:46:08.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.640295 29938 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:46:08.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.140187 30019 master_runner.cc:186] Error getting master registration for 127.29.53.62:41263: OK, instance_id { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" instance_seqno: 1761756363121017 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
16:46:08.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.693068 30019 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:46:08.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.735173 30068 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b76f6b3da73b432fb8397d6dd53510c1 in term 1.
16:46:08.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.746847 30222 sys_catalog.cc:455] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b76f6b3da73b432fb8397d6dd53510c1. Latest consensus state: current_term: 1 leader_uuid: "b76f6b3da73b432fb8397d6dd53510c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:08.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.747634 30222 sys_catalog.cc:458] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: This master's current role is: LEADER
16:46:08.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:08.795069 30237 catalog_manager.cc:1568] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
16:46:08.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:08.807580 30237 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
16:46:08.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.808607 30238 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
16:46:08.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.818152 30238 catalog_manager.cc:1494] Initializing Kudu cluster ID...
16:46:08.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.880100 30068 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 1 FOLLOWER]: Refusing update from remote peer b76f6b3da73b432fb8397d6dd53510c1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:46:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.882153 30220 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:46:08.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.886675 29985 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Refusing update from remote peer b76f6b3da73b432fb8397d6dd53510c1: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
16:46:08.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.889348 30220 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
16:46:08.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.950713 30241 sys_catalog.cc:455] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b76f6b3da73b432fb8397d6dd53510c1. Latest consensus state: current_term: 1 leader_uuid: "b76f6b3da73b432fb8397d6dd53510c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:08.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.951573 30241 sys_catalog.cc:458] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: This master's current role is: FOLLOWER
16:46:08.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.957131 30240 mvcc.cc:204] Tried to move back new op lower bound from 7216154086847168512 to 7216154086145175552. Current Snapshot: MvccSnapshot[applied={T|T < 7216154086847168512}]
16:46:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.967883 30222 sys_catalog.cc:455] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b76f6b3da73b432fb8397d6dd53510c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:08.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.968997 30222 sys_catalog.cc:458] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: This master's current role is: LEADER
16:46:08.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.983733 30241 sys_catalog.cc:455] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b76f6b3da73b432fb8397d6dd53510c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:08.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.984721 30241 sys_catalog.cc:458] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: This master's current role is: FOLLOWER
16:46:08.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.985845 30244 mvcc.cc:204] Tried to move back new op lower bound from 7216154086847168512 to 7216154086145175552. Current Snapshot: MvccSnapshot[applied={T|T < 7216154086847168512 or (T in {7216154086847168512})}]
16:46:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.988793 30238 catalog_manager.cc:1357] Generated new cluster ID: 524b096ae3d14ebea944ac34909d7ad6
16:46:08.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.989243 30238 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
16:46:08.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.996948 30242 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b76f6b3da73b432fb8397d6dd53510c1. Latest consensus state: current_term: 1 leader_uuid: "b76f6b3da73b432fb8397d6dd53510c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:08.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:08.997648 30242 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: This master's current role is: FOLLOWER
16:46:09.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.005992 30220 sys_catalog.cc:455] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b76f6b3da73b432fb8397d6dd53510c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:09.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.006704 30220 sys_catalog.cc:458] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: This master's current role is: LEADER
16:46:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.015300 30242 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b76f6b3da73b432fb8397d6dd53510c1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:09.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.016059 30242 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: This master's current role is: FOLLOWER
16:46:09.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.027050 30238 catalog_manager.cc:1380] Generated new certificate authority record
16:46:09.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.029527 30238 catalog_manager.cc:1514] Loading token signing keys...
16:46:09.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.085727 30238 catalog_manager.cc:6022] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1: Generated new TSK 0
16:46:09.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.086961 30238 catalog_manager.cc:1524] Initializing in-progress tserver states...
16:46:09.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.53.1@KRBTEST.COM; defaulting to no policy
16:46:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:09.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.53.1@KRBTEST.COM" created.
16:46:09.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:09.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.29.53.1@KRBTEST.COM; defaulting to no policy
16:46:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:09.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.29.53.1@KRBTEST.COM" created.
16:46:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.255966 30215 catalog_manager.cc:1269] Loaded cluster ID: 524b096ae3d14ebea944ac34909d7ad6
16:46:09.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.256242 30215 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: loading cluster ID for follower catalog manager: success
16:46:09.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.273754 30215 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: acquiring CA information for follower catalog manager: success
16:46:09.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.280095 30215 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:09.295677 29908 mini_kdc.cc:279] Time spent creating service keytab for HTTP/127.29.53.1: real 0.127s user 0.000s sys 0.015s
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.298308 29908 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/wal
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/data
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/logs
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/data/info.pb
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:46:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.1:0
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.53.1
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.1
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.1
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:09.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:46:09.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.514891 30221 catalog_manager.cc:1269] Loaded cluster ID: 524b096ae3d14ebea944ac34909d7ad6
16:46:09.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.516142 30221 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: loading cluster ID for follower catalog manager: success
16:46:09.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.537290 30221 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: acquiring CA information for follower catalog manager: success
16:46:09.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:09.545156 30221 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:46:09.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:09.999253 30256 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:46:10.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:09.999941 30256 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:46:10.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:10.001030 30256 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:46:10.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:10.071952 30256 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:46:10.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:10.074447 30256 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.53.1
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:10.145965 30256 tablet_server_runner.cc:78] Tablet server non-default flags:
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/data
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/wal
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.1
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.1:0
16:46:10.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/data/info.pb
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.1
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.53.1
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/logs
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:46:10.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:46:10.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:10.147784 30256 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:46:10.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:10.150790 30256 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:46:10.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:10.187379 30262 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:11.575872 30261 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 30256
16:46:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:12.827262 30256 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.641s user 0.616s sys 1.007s
16:46:12.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:12.827631 30256 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.641s user 0.616s sys 1.007s
16:46:12.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:10.189625 30263 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:12.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:12.835351 30272 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.836402 30256 server_base.cc:1047] running on GCE node
16:46:12.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.838050 30256 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:46:12.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.840902 30256 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:46:12.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.842393 30256 hybrid_clock.cc:648] HybridClock initialized: now 1761756372842258 us; error 120 us; skew 500 ppm
16:46:12.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.843519 30256 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:46:12.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:12 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756372, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:12.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.853600 30256 init.cc:377] Logged in from keytab as kudu/127.29.53.1@KRBTEST.COM (short username kudu)
16:46:12.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.861407 30256 webserver.cc:492] Webserver started at http://127.29.53.1:37461/ using document root <none> and password file <none>
16:46:12.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.862818 30256 fs_manager.cc:362] Metadata directory not provided
16:46:12.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.863288 30256 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:46:12.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.863818 30256 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:46:12.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.870787 30256 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/data/instance:
16:46:12.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f78b246a4c5d458bb7ee8dae100d3b83"
16:46:12.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:12 on dist-test-slave-wkbt"
16:46:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.872316 30256 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/wal/instance:
16:46:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f78b246a4c5d458bb7ee8dae100d3b83"
16:46:12.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:12 on dist-test-slave-wkbt"
16:46:12.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.884377 30256 fs_manager.cc:696] Time spent creating directory manager: real 0.011s user 0.009s sys 0.001s
16:46:12.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.893416 30279 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:46:12.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.894654 30256 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.004s
16:46:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.895018 30256 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/wal
16:46:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f78b246a4c5d458bb7ee8dae100d3b83"
16:46:12.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:12 on dist-test-slave-wkbt"
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.895421 30256 fs_report.cc:389] FS layout report
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/wal
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/wal
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/data/data
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:46:12.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:46:12.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:12 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756372, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:12.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.959609 29953 init.cc:295] Successfully reacquired a new kerberos TGT
16:46:12.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.959923 30256 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:46:12.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.971078 30256 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:46:12.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.971665 30256 kserver.cc:163] Server-wide thread pool size limit: 3276
16:46:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.974973 30256 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:46:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.982748 30256 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:46:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.982985 30256 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:46:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.983325 30256 ts_tablet_manager.cc:616] Registered 0 tablets
16:46:12.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:12.983544 30256 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:46:13.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.257176 30256 rpc_server.cc:307] RPC server started. Bound to: 127.29.53.1:34725
16:46:13.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.257612 30392 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.53.1:34725 every 8 connection(s)
16:46:13.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.261550 30256 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/data/info.pb
16:46:13.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.264691 29908 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 30256
16:46:13.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.265250 29908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-0/wal/instance
16:46:13.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:13 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756372, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.1@KRBTEST.COM for kudu/127.29.53.62@KRBTEST.COM
16:46:13.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:13 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756372, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.1@KRBTEST.COM for kudu/127.29.53.61@KRBTEST.COM
16:46:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.53.2@KRBTEST.COM; defaulting to no policy
16:46:13.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:13 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756372, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.1@KRBTEST.COM for kudu/127.29.53.60@KRBTEST.COM
16:46:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.53.2@KRBTEST.COM" created.
16:46:13.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.362568 30394 heartbeater.cc:344] Connected to a master server at 127.29.53.61:44169
16:46:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.363334 30394 heartbeater.cc:461] Registering TS with master...
16:46:13.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.366132 30394 heartbeater.cc:507] Master 127.29.53.61:44169 requested a full tablet report, sending...
16:46:13.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.369426 30057 ts_manager.cc:194] Registered new tserver with Master: f78b246a4c5d458bb7ee8dae100d3b83 (127.29.53.1:34725)
16:46:13.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.372637 30397 heartbeater.cc:344] Connected to a master server at 127.29.53.62:41263
16:46:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.373068 30397 heartbeater.cc:461] Registering TS with master...
16:46:13.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.374306 30393 heartbeater.cc:344] Connected to a master server at 127.29.53.60:38873
16:46:13.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.374778 30393 heartbeater.cc:461] Registering TS with master...
16:46:13.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.374845 30397 heartbeater.cc:507] Master 127.29.53.62:41263 requested a full tablet report, sending...
16:46:13.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.376312 30393 heartbeater.cc:507] Master 127.29.53.60:38873 requested a full tablet report, sending...
16:46:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.378854 30145 ts_manager.cc:194] Registered new tserver with Master: f78b246a4c5d458bb7ee8dae100d3b83 (127.29.53.1:34725)
16:46:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.378852 29974 ts_manager.cc:194] Registered new tserver with Master: f78b246a4c5d458bb7ee8dae100d3b83 (127.29.53.1:34725)
16:46:13.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.381804 30145 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.29.53.1@KRBTEST.COM'} at 127.29.53.1:56447
16:46:13.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:13.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:13.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:13.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:13.395466 29908 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.29.53.2: real 0.120s user 0.002s sys 0.040s
16:46:13.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.29.53.2@KRBTEST.COM; defaulting to no policy
16:46:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:13.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.29.53.2@KRBTEST.COM" created.
16:46:13.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:13.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:13.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.443923 29908 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/wal
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/data
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/logs
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/data/info.pb
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:46:13.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.2:0
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.53.2
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.2
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.2
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:13.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:46:13.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:13.803290 30407 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:46:13.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:13.803779 30407 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:46:13.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:13.804380 30407 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:46:13.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:13.839248 30407 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:46:13.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:13.840262 30407 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.53.2
16:46:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.882261 30407 tablet_server_runner.cc:78] Tablet server non-default flags:
16:46:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/data
16:46:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/wal
16:46:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.2
16:46:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:13.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.2:0
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/data/info.pb
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.2
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.53.2
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/logs
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:46:13.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.883584 30407 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:46:13.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:13.884614 30407 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:46:13.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:13.897822 30415 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:14.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:14.386210 30393 heartbeater.cc:499] Master 127.29.53.60:38873 was elected leader, sending a full tablet report...
16:46:15.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:15 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756375, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:15.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.100737 30036 init.cc:295] Successfully reacquired a new kerberos TGT
16:46:15.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:13.900025 30416 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:15.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:15.247752 30418 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:15.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:15.251530 30417 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1347 milliseconds
16:46:15.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.251710 30407 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:46:15.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.253343 30407 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:46:15.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.256618 30407 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:46:15.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.258317 30407 hybrid_clock.cc:648] HybridClock initialized: now 1761756375258264 us; error 47 us; skew 500 ppm
16:46:15.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.259518 30407 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:46:15.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:15 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756375, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:15.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.267624 30407 init.cc:377] Logged in from keytab as kudu/127.29.53.2@KRBTEST.COM (short username kudu)
16:46:15.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.274402 30407 webserver.cc:492] Webserver started at http://127.29.53.2:43347/ using document root <none> and password file <none>
16:46:15.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.275504 30407 fs_manager.cc:362] Metadata directory not provided
16:46:15.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.275727 30407 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:46:15.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.276273 30407 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:46:15.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.280601 30407 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/data/instance:
16:46:15.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17db4142df3a468384b9d16f6c2bd85a"
16:46:15.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:15 on dist-test-slave-wkbt"
16:46:15.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.281752 30407 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/wal/instance:
16:46:15.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17db4142df3a468384b9d16f6c2bd85a"
16:46:15.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:15 on dist-test-slave-wkbt"
16:46:15.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.290494 30407 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.004s
16:46:15.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.295923 30432 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:46:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.296958 30407 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
16:46:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.297287 30407 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/wal
16:46:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "17db4142df3a468384b9d16f6c2bd85a"
16:46:15.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:15 on dist-test-slave-wkbt"
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.297613 30407 fs_report.cc:389] FS layout report
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/wal
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/wal
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/data/data
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:46:15.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:46:15.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:46:15.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.356496 30407 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:46:15.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.362391 30407 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:46:15.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.363125 30407 kserver.cc:163] Server-wide thread pool size limit: 3276
16:46:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.366308 30407 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:46:15.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.370935 30407 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:46:15.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.371162 30407 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:46:15.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.371491 30407 ts_tablet_manager.cc:616] Registered 0 tablets
16:46:15.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.371659 30407 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:46:15.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.637012 30407 rpc_server.cc:307] RPC server started. Bound to: 127.29.53.2:38379
16:46:15.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.637482 30544 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.53.2:38379 every 8 connection(s)
16:46:15.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.640036 30407 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/data/info.pb
16:46:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.641633 29908 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 30407
16:46:15.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.642170 29908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-1/wal/instance
16:46:15.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:15.728708 29918 outbound_call.cc:321] RPC callback for RPC call kudu.master.MasterService.ConnectToMaster -> {remote=127.29.53.60:38873, user_credentials={real_user=slave}} blocked reactor thread for 46635.5us
16:46:15.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:15 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756375, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.2@KRBTEST.COM for kudu/127.29.53.60@KRBTEST.COM
16:46:15.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:15 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756375, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.2@KRBTEST.COM for kudu/127.29.53.61@KRBTEST.COM
16:46:15.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.29.53.3@KRBTEST.COM; defaulting to no policy
16:46:15.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:15 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756375, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.2@KRBTEST.COM for kudu/127.29.53.62@KRBTEST.COM
16:46:15.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:15.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.29.53.3@KRBTEST.COM" created.
16:46:15.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.769500 30545 heartbeater.cc:344] Connected to a master server at 127.29.53.60:38873
16:46:15.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.770046 30545 heartbeater.cc:461] Registering TS with master...
16:46:15.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.772158 30545 heartbeater.cc:507] Master 127.29.53.60:38873 requested a full tablet report, sending...
16:46:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.773531 30546 heartbeater.cc:344] Connected to a master server at 127.29.53.61:44169
16:46:15.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.774015 30546 heartbeater.cc:461] Registering TS with master...
16:46:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.774943 30145 ts_manager.cc:194] Registered new tserver with Master: 17db4142df3a468384b9d16f6c2bd85a (127.29.53.2:38379)
16:46:15.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.775530 30546 heartbeater.cc:507] Master 127.29.53.61:44169 requested a full tablet report, sending...
16:46:15.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.779618 30057 ts_manager.cc:194] Registered new tserver with Master: 17db4142df3a468384b9d16f6c2bd85a (127.29.53.2:38379)
16:46:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.782608 30548 heartbeater.cc:344] Connected to a master server at 127.29.53.62:41263
16:46:15.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.782953 30548 heartbeater.cc:461] Registering TS with master...
16:46:15.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.783985 30548 heartbeater.cc:507] Master 127.29.53.62:41263 requested a full tablet report, sending...
16:46:15.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.786267 30145 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.29.53.2@KRBTEST.COM'} at 127.29.53.2:55297
16:46:15.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:15.789378 29974 ts_manager.cc:194] Registered new tserver with Master: 17db4142df3a468384b9d16f6c2bd85a (127.29.53.2:38379)
16:46:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:15.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.29.53.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:16.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:16.790721 30545 heartbeater.cc:499] Master 127.29.53.60:38873 was elected leader, sending a full tablet report...
16:46:17.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:15.799438 29908 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.29.53.3: real 0.116s user 0.002s sys 0.028s
16:46:17.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.29.53.3@KRBTEST.COM; defaulting to no policy
16:46:17.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:17.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.29.53.3@KRBTEST.COM" created.
16:46:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
16:46:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:17.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.29.53.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab.
16:46:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:17.093336 29908 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/wal
16:46:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/data
16:46:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:46:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:46:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:46:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:17.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/logs
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/data/info.pb
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.3:0
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.53.3
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.3
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.3
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:17.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:46:17.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:17.485715 30564 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:46:17.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:17.486250 30564 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:46:17.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:17.486812 30564 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:46:17.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:17 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756377, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:17.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:17.512853 30124 init.cc:295] Successfully reacquired a new kerberos TGT
16:46:17.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:17.528992 30564 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:46:17.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:17.529922 30564 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.53.3
16:46:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:17.568459 30564 tablet_server_runner.cc:78] Tablet server non-default flags:
16:46:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/data
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/wal
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.3
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.3:0
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/data/info.pb
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.3
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:17.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.53.3
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/logs
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:46:17.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:17.569814 30564 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:46:17.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:17.571089 30564 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:46:17.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:17.585752 30570 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:19.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:19.174836 29893 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5540 after lost signal to thread 29767
16:46:19.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:19.176013 29893 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5680 after lost signal to thread 29896
16:46:19.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:19.001186 30569 debug-util.cc:398] Leaking SignalData structure 0x7b080002ee00 after lost signal to thread 30564
16:46:19.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:19.559913 30569 kernel_stack_watchdog.cc:198] Thread 30564 stuck at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:642 for 404ms:
16:46:19.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
16:46:19.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
16:46:19.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
16:46:19.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
16:46:19.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:17.589587 30571 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:19.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:19.566083 30573 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:19.566694 30564 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.976s user 0.606s sys 1.271s
16:46:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:19.567054 30564 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.978s user 0.606s sys 1.271s
16:46:19.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.579622 30564 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
16:46:19.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:19.580153 30572 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1990 milliseconds
16:46:19.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.581553 30564 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:46:19.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.584568 30564 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:46:19.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.585990 30564 hybrid_clock.cc:648] HybridClock initialized: now 1761756379585839 us; error 161 us; skew 500 ppm
16:46:19.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.587086 30564 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:46:19.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:19 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756379, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:19.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.594628 30564 init.cc:377] Logged in from keytab as kudu/127.29.53.3@KRBTEST.COM (short username kudu)
16:46:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.601169 30564 webserver.cc:492] Webserver started at http://127.29.53.3:39421/ using document root <none> and password file <none>
16:46:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.602090 30564 fs_manager.cc:362] Metadata directory not provided
16:46:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.602304 30564 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:46:19.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.602689 30564 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
16:46:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.606774 30564 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/data/instance:
16:46:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28fceda62ce942b68bab0f8c70c336e4"
16:46:19.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:19 on dist-test-slave-wkbt"
16:46:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.607798 30564 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/wal/instance:
16:46:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28fceda62ce942b68bab0f8c70c336e4"
16:46:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:19 on dist-test-slave-wkbt"
16:46:19.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.615875 30564 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.002s
16:46:19.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.622534 30586 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:46:19.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.623839 30564 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.003s
16:46:19.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.624173 30564 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/wal
16:46:19.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "28fceda62ce942b68bab0f8c70c336e4"
16:46:19.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:19 on dist-test-slave-wkbt"
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.624504 30564 fs_report.cc:389] FS layout report
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/wal
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/wal
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/data/data
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:46:19.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:46:19.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.736347 30564 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:46:19.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.750051 30564 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:46:19.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.750818 30564 kserver.cc:163] Server-wide thread pool size limit: 3276
16:46:19.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.754858 30564 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
16:46:19.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.768208 30564 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
16:46:19.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.768518 30564 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
16:46:19.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.768823 30564 ts_tablet_manager.cc:616] Registered 0 tablets
16:46:19.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:19.769052 30564 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
16:46:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:20.164858 29252 debug-util.cc:398] Leaking SignalData structure 0x7b0800076280 after lost signal to thread 29189
16:46:20.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:20.165745 29252 debug-util.cc:398] Leaking SignalData structure 0x7b08000738e0 after lost signal to thread 29255
16:46:20.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.241354 30564 rpc_server.cc:307] RPC server started. Bound to: 127.29.53.3:43235
16:46:20.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.241506 30700 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.53.3:43235 every 8 connection(s)
16:46:20.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.246070 30564 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/data/info.pb
16:46:20.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.251035 29908 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 30564
16:46:20.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.252063 29908 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/ts-2/wal/instance
16:46:20.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:20 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756379, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.3@KRBTEST.COM for kudu/127.29.53.60@KRBTEST.COM
16:46:20.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:20 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756379, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.3@KRBTEST.COM for kudu/127.29.53.61@KRBTEST.COM
16:46:20.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:20 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756379, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.3@KRBTEST.COM for kudu/127.29.53.62@KRBTEST.COM
16:46:20.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.470330 30701 heartbeater.cc:344] Connected to a master server at 127.29.53.60:38873
16:46:20.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.470883 30701 heartbeater.cc:461] Registering TS with master...
16:46:20.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.472421 30701 heartbeater.cc:507] Master 127.29.53.60:38873 requested a full tablet report, sending...
16:46:20.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.479499 30145 ts_manager.cc:194] Registered new tserver with Master: 28fceda62ce942b68bab0f8c70c336e4 (127.29.53.3:43235)
16:46:20.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.489820 30145 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.29.53.3@KRBTEST.COM'} at 127.29.53.3:39167
16:46:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.493602 30704 heartbeater.cc:344] Connected to a master server at 127.29.53.61:44169
16:46:20.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.494005 30704 heartbeater.cc:461] Registering TS with master...
16:46:20.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.495591 30704 heartbeater.cc:507] Master 127.29.53.61:44169 requested a full tablet report, sending...
16:46:20.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.503835 30057 ts_manager.cc:194] Registered new tserver with Master: 28fceda62ce942b68bab0f8c70c336e4 (127.29.53.3:43235)
16:46:20.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.509696 30706 heartbeater.cc:344] Connected to a master server at 127.29.53.62:41263
16:46:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.510144 30706 heartbeater.cc:461] Registering TS with master...
16:46:20.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.510798 30706 heartbeater.cc:507] Master 127.29.53.62:41263 requested a full tablet report, sending...
16:46:20.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.513661 29974 ts_manager.cc:194] Registered new tserver with Master: 28fceda62ce942b68bab0f8c70c336e4 (127.29.53.3:43235)
16:46:20.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.528873 29908 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
16:46:20.564 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:46:20.566 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
16:46:20.580 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
16:46:20.581 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
16:46:20.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.29.53.62"
port: 41263
}
bound_http_address {
host: "127.29.53.62"
port: 40455
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.29.53.61"
port: 44169
}
bound_http_address {
host: "127.29.53.61"
port: 35445
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.29.53.60"
port: 38873
}
bound_http_address {
host: "127.29.53.60"
port: 46741
}
}
}
16:46:20.587 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
16:46:20.589 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.29.53.1"
port: 34725
}
bound_http_address {
host: "127.29.53.1"
port: 37461
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.29.53.2"
port: 38379
}
bound_http_address {
host: "127.29.53.2"
port: 43347
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.29.53.3"
port: 43235
}
bound_http_address {
host: "127.29.53.3"
port: 39421
}
}
}
16:46:20.589 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:46:20.591 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:46:20.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:20 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756380, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:46:20.664 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:46:20.669 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc
16:46:20.684 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:46:20.687 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc
16:46:20.690 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:46:20.695 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
16:46:20.698 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
16:46:20.698 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
16:46:20.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:20 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756380, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:20.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
16:46:20.757 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:46:20.757 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc
16:46:20.759 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
16:46:20.760 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.53.62:41263
16:46:20.764 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
16:46:20.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.765548 29908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 29938
16:46:20.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:20.820170 30434 connection.cc:537] client connection to 127.29.53.62:41263 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
16:46:20.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:20.821928 30548 heartbeater.cc:646] Failed to heartbeat to 127.29.53.62:41263 (0 consecutive failures): Network error: Failed to send heartbeat to master: recv error from unknown peer: Transport endpoint is not connected (error 107)
16:46:20.825 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:46:20.825 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.53.61:44169
16:46:20.826 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
16:46:20.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.829739 29908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 30019
16:46:20.881 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:46:20.882 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.29.53.60:38873
16:46:20.882 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
16:46:20.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.883297 29908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 30102
16:46:20.919 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:46:20.919 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.53.62:41263
16:46:20.923 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
16:46:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:20.926100 29908 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal
16:46:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data
16:46:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:46:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:46:20.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/logs
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data/info.pb
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.62:41263
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.62
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40455
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.62
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:20.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:46:20.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:46:21.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.430855 30397 heartbeater.cc:646] Failed to heartbeat to 127.29.53.62:41263 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.29.53.62:41263: connect: Connection refused (error 111)
16:46:21.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.450141 30716 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:46:21.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.450984 30716 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:46:21.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.452348 30716 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:46:21.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:21.502120 30701 heartbeater.cc:499] Master 127.29.53.60:38873 was elected leader, sending a full tablet report...
16:46:21.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.526306 30706 heartbeater.cc:646] Failed to heartbeat to 127.29.53.62:41263 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.29.53.62:41263: connect: Connection refused (error 111)
16:46:21.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.545699 30716 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:46:21.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.546041 30716 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:46:21.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.546551 30716 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:46:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.546906 30716 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:46:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.547284 30716 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:46:21.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.547641 30716 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:46:21.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.547907 30716 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38345/jwks.json
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:21.630897 30716 master_runner.cc:386] Master server non-default flags:
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.62
16:46:21.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.62:41263
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data/info.pb
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.62
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40455
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/logs
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:46:21.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:46:21.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:21.632884 30716 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:46:21.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:21.634500 30716 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:46:21.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.655628 30727 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:22.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:22 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756382, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:22.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:22.860423 30275 init.cc:295] Successfully reacquired a new kerberos TGT
16:46:23.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.055716 30725 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 30716
16:46:23.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:21.655617 30726 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:23.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.091332 30716 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.438s user 0.506s sys 0.814s
16:46:23.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.091775 30716 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.439s user 0.506s sys 0.815s
16:46:23.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.101824 30742 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:23.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.101838 30716 server_base.cc:1047] running on GCE node
16:46:23.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.103504 30716 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:46:23.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.106492 30716 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:46:23.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.107944 30716 hybrid_clock.cc:648] HybridClock initialized: now 1761756383107858 us; error 48 us; skew 500 ppm
16:46:23.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.108884 30716 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:46:23.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:23 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756383, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:23.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.115048 30716 init.cc:377] Logged in from keytab as kudu/127.29.53.62@KRBTEST.COM (short username kudu)
16:46:23.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.120529 30716 webserver.cc:492] Webserver started at http://127.29.53.62:40455/ using document root <none> and password file <none>
16:46:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.121512 30716 fs_manager.cc:362] Metadata directory not provided
16:46:23.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.121747 30716 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:46:23.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.129632 30716 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.005s sys 0.000s
16:46:23.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.134212 30749 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:46:23.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.135229 30716 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
16:46:23.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.135564 30716 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1869e0a1407440e79e0b04ecb71da9b0"
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:02 on dist-test-slave-wkbt"
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.137892 30716 fs_report.cc:389] FS layout report
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/wal
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data/data
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:46:23.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:46:23.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.193768 30716 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:46:23.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.199311 30716 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:46:23.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.199726 30716 kserver.cc:163] Server-wide thread pool size limit: 3276
16:46:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.280723 30716 rpc_server.cc:307] RPC server started. Bound to: 127.29.53.62:41263
16:46:23.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.280827 30800 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.53.62:41263 every 8 connection(s)
16:46:23.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.283527 30716 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-0/data/info.pb
16:46:23.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.285356 29908 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 30716
16:46:23.286 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:46:23.286 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.53.61:44169
16:46:23.286 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
16:46:23.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.288867 29908 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:23.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:23.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal
16:46:23.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data
16:46:23.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:46:23.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:46:23.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/logs
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data/info.pb
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.61:44169
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.61
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35445
16:46:23.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.61
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:46:23.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:46:23.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.301513 30801 sys_catalog.cc:263] Verifying existing consensus state
16:46:23.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.311811 30801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: Bootstrap starting.
16:46:23.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.368425 30801 log.cc:826] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: Log is configured to *not* fsync() on all Append() calls
16:46:23.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.388926 30801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:46:23.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.390002 30801 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: Bootstrap complete.
16:46:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.415190 30801 raft_consensus.cc:359] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:23.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.417713 30801 raft_consensus.cc:740] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1869e0a1407440e79e0b04ecb71da9b0, State: Initialized, Role: FOLLOWER
16:46:23.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.418462 30801 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.421155 30805 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:23.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.422094 30805 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: This master's current role is: FOLLOWER
16:46:23.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.422469 30801 sys_catalog.cc:565] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: configured and running, proceeding with master startup.
16:46:23.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.476970 30816 catalog_manager.cc:1269] Loaded cluster ID: 524b096ae3d14ebea944ac34909d7ad6
16:46:23.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.477254 30816 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: loading cluster ID for follower catalog manager: success
16:46:23.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.485045 30816 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: acquiring CA information for follower catalog manager: success
16:46:23.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:23 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756383, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.62@KRBTEST.COM for kudu/127.29.53.62@KRBTEST.COM
16:46:23.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:23 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756382, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.1@KRBTEST.COM for kudu/127.29.53.62@KRBTEST.COM
16:46:23.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.490959 30816 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:46:23.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.515255 30397 heartbeater.cc:344] Connected to a master server at 127.29.53.62:41263
16:46:23.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.522089 30766 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "f78b246a4c5d458bb7ee8dae100d3b83" instance_seqno: 1761756373196543) as {username='kudu', principal='kudu/127.29.53.1@KRBTEST.COM'} at 127.29.53.1:55725; Asking this server to re-register.
16:46:23.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.523548 30397 heartbeater.cc:461] Registering TS with master...
16:46:23.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.527297 30766 ts_manager.cc:194] Registered new tserver with Master: f78b246a4c5d458bb7ee8dae100d3b83 (127.29.53.1:34725)
16:46:23.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.525427 30716 master_runner.cc:186] Error getting master registration for 127.29.53.61:44169: Network error: Client connection negotiation failed: client connection to 127.29.53.61:44169: connect: Connection refused (error 111),
16:46:23.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.551821 30716 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:46:23.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.595672 30706 heartbeater.cc:344] Connected to a master server at 127.29.53.62:41263
16:46:23.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.598380 30766 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "28fceda62ce942b68bab0f8c70c336e4" instance_seqno: 1761756380129724) as {username='kudu', principal='kudu/127.29.53.3@KRBTEST.COM'} at 127.29.53.3:51723; Asking this server to re-register.
16:46:23.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.599601 30706 heartbeater.cc:461] Registering TS with master...
16:46:23.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.602517 30766 ts_manager.cc:194] Registered new tserver with Master: 28fceda62ce942b68bab0f8c70c336e4 (127.29.53.3:43235)
16:46:23.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.794734 30803 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:46:23.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.795552 30803 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:46:23.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.796708 30803 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:46:23.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.855401 30803 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:46:23.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.855744 30803 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:46:23.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.856240 30803 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:46:23.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.856554 30803 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:46:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.856870 30803 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:46:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.857241 30803 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:46:23.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.857472 30803 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38345/jwks.json
16:46:23.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.887215 30548 heartbeater.cc:344] Connected to a master server at 127.29.53.62:41263
16:46:23.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.892006 30766 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "17db4142df3a468384b9d16f6c2bd85a" instance_seqno: 1761756375566216) as {username='kudu', principal='kudu/127.29.53.2@KRBTEST.COM'} at 127.29.53.2:47607; Asking this server to re-register.
16:46:23.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.894372 30548 heartbeater.cc:461] Registering TS with master...
16:46:23.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.899187 30766 ts_manager.cc:194] Registered new tserver with Master: 17db4142df3a468384b9d16f6c2bd85a (127.29.53.2:38379)
16:46:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.925047 30803 master_runner.cc:386] Master server non-default flags:
16:46:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data
16:46:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal
16:46:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:46:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:23.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.61
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.61:44169
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data/info.pb
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.61
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35445
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/logs
16:46:23.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:46:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:46:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:46:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:46:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:46:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:46:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:46:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:46:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:46:23.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.926700 30803 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:46:23.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:23.928056 30803 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:46:23.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.948594 30836 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:24.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:24.628151 30844 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:46:24.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:24.629454 30844 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:24.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:24.636198 30844 leader_election.cc:290] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 083204b027784cb697b1041fa88ef693 (127.29.53.61:44169), b76f6b3da73b432fb8397d6dd53510c1 (127.29.53.60:38873)
16:46:24.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:24.654608 30753 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.53.61:44169: connect: Connection refused (error 111)
16:46:24.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:24.667045 30753 leader_election.cc:336] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 083204b027784cb697b1041fa88ef693 (127.29.53.61:44169): Network error: Client connection negotiation failed: client connection to 127.29.53.61:44169: connect: Connection refused (error 111)
16:46:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:24.669862 30753 leader_election.cc:336] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b76f6b3da73b432fb8397d6dd53510c1 (127.29.53.60:38873): Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:24.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:24.670269 30753 leader_election.cc:304] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 1869e0a1407440e79e0b04ecb71da9b0; no voters: 083204b027784cb697b1041fa88ef693, b76f6b3da73b432fb8397d6dd53510c1
16:46:24.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:24.671022 30844 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
16:46:25.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:25 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756385, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:25.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.273300 30428 init.cc:295] Successfully reacquired a new kerberos TGT
16:46:25.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:25.351758 30835 debug-util.cc:398] Leaking SignalData structure 0x7b0800027080 after lost signal to thread 30803
16:46:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:25.471920 30803 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.524s user 0.487s sys 0.942s
16:46:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:23.949604 30837 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:25.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:25.472299 30803 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.525s user 0.487s sys 0.942s
16:46:25.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.480432 30803 server_base.cc:1047] running on GCE node
16:46:25.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:25.481298 30851 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:25.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.482769 30803 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:46:25.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.485755 30803 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:46:25.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:25.487069 29409 debug-util.cc:398] Leaking SignalData structure 0x7b080008a5a0 after lost signal to thread 29338
16:46:25.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.487272 30803 hybrid_clock.cc:648] HybridClock initialized: now 1761756385487140 us; error 96 us; skew 500 ppm
16:46:25.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.488734 30803 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:46:25.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:25.488689 29409 debug-util.cc:398] Leaking SignalData structure 0x7b08000b25e0 after lost signal to thread 29412
16:46:25.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:25 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756385, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:25.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.496886 30803 init.cc:377] Logged in from keytab as kudu/127.29.53.61@KRBTEST.COM (short username kudu)
16:46:25.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.516198 30803 webserver.cc:492] Webserver started at http://127.29.53.61:35445/ using document root <none> and password file <none>
16:46:25.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.517542 30803 fs_manager.cc:362] Metadata directory not provided
16:46:25.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.517861 30803 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:46:25.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.549497 30803 fs_manager.cc:714] Time spent opening directory manager: real 0.027s user 0.007s sys 0.019s
16:46:25.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.568128 30860 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:46:25.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.569543 30803 fs_manager.cc:730] Time spent opening block manager: real 0.017s user 0.008s sys 0.007s
16:46:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.569990 30803 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal
16:46:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "083204b027784cb697b1041fa88ef693"
16:46:25.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:05 on dist-test-slave-wkbt"
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.572824 30803 fs_report.cc:389] FS layout report
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/wal
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data/data
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:46:25.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:46:25.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:46:25.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:46:25.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:46:25.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.669518 30803 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:46:25.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.683357 30803 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:46:25.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.683984 30803 kserver.cc:163] Server-wide thread pool size limit: 3276
16:46:25.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.905628 30803 rpc_server.cc:307] RPC server started. Bound to: 127.29.53.61:44169
16:46:25.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.905721 30913 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.53.61:44169 every 8 connection(s)
16:46:25.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.911401 30803 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-1/data/info.pb
16:46:25.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.914119 29908 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 30803
16:46:25.915 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:46:25.916 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.29.53.60:38873
16:46:25.916 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
16:46:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.918324 29908 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu
16:46:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal
16:46:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data
16:46:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
16:46:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
16:46:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
16:46:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:25.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/logs
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data/info.pb
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.60:38873
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.60
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46741
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.60
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
16:46:25.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
16:46:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
16:46:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
16:46:25.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
16:46:25.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.969743 30914 sys_catalog.cc:263] Verifying existing consensus state
16:46:25.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:25.977919 30914 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: Bootstrap starting.
16:46:26.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:26 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756385, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.2@KRBTEST.COM for kudu/127.29.53.61@KRBTEST.COM
16:46:26.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.036748 30914 log.cc:826] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: Log is configured to *not* fsync() on all Append() calls
16:46:26.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.039816 30546 heartbeater.cc:344] Connected to a master server at 127.29.53.61:44169
16:46:26.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.072072 30914 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:46:26.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.073619 30914 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: Bootstrap complete.
16:46:26.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.106137 30914 raft_consensus.cc:359] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:26.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.109743 30914 raft_consensus.cc:740] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 083204b027784cb697b1041fa88ef693, State: Initialized, Role: FOLLOWER
16:46:26.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.110945 30914 consensus_queue.cc:260] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:26.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.128506 30921 sys_catalog.cc:455] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:26.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.129403 30921 sys_catalog.cc:458] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: This master's current role is: FOLLOWER
16:46:26.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.131731 30914 sys_catalog.cc:565] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: configured and running, proceeding with master startup.
16:46:26.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.174732 30389 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9100 after lost signal to thread 30257
16:46:26.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.176017 30389 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae1c0 after lost signal to thread 30392
16:46:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.302271 30932 catalog_manager.cc:1269] Loaded cluster ID: 524b096ae3d14ebea944ac34909d7ad6
16:46:26.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.302559 30932 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: loading cluster ID for follower catalog manager: success
16:46:26.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.318807 30932 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: acquiring CA information for follower catalog manager: success
16:46:26.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:26 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756385, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.61@KRBTEST.COM for kudu/127.29.53.62@KRBTEST.COM
16:46:26.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.338667 30932 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
16:46:26.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.387265 30803 master_runner.cc:186] Error getting master registration for 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111),
16:46:26.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.566757 30803 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:46:26.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.571646 30394 heartbeater.cc:344] Connected to a master server at 127.29.53.61:44169
16:46:26.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.577960 30879 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "f78b246a4c5d458bb7ee8dae100d3b83" instance_seqno: 1761756373196543) as {username='kudu', principal='kudu/127.29.53.1@KRBTEST.COM'} at 127.29.53.1:47921; Asking this server to re-register.
16:46:26.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.579349 30394 heartbeater.cc:461] Registering TS with master...
16:46:26.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.583885 30879 ts_manager.cc:194] Registered new tserver with Master: f78b246a4c5d458bb7ee8dae100d3b83 (127.29.53.1:34725)
16:46:26.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.667330 30704 heartbeater.cc:344] Connected to a master server at 127.29.53.61:44169
16:46:26.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.670038 30879 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "28fceda62ce942b68bab0f8c70c336e4" instance_seqno: 1761756380129724) as {username='kudu', principal='kudu/127.29.53.3@KRBTEST.COM'} at 127.29.53.3:37021; Asking this server to re-register.
16:46:26.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.671310 30704 heartbeater.cc:461] Registering TS with master...
16:46:26.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.674192 30879 ts_manager.cc:194] Registered new tserver with Master: 28fceda62ce942b68bab0f8c70c336e4 (127.29.53.3:43235)
16:46:26.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.733809 30915 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
16:46:26.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.734630 30915 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
16:46:26.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.735947 30915 flags.cc:425] Enabled unsafe flag: --never_fsync=true
16:46:26.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.762586 30944 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
16:46:26.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.763110 30944 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:26.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.772522 30944 leader_election.cc:290] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 083204b027784cb697b1041fa88ef693 (127.29.53.61:44169), b76f6b3da73b432fb8397d6dd53510c1 (127.29.53.60:38873)
16:46:26.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.818852 30753 leader_election.cc:336] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b76f6b3da73b432fb8397d6dd53510c1 (127.29.53.60:38873): Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:26.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.829800 30889 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1869e0a1407440e79e0b04ecb71da9b0" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "083204b027784cb697b1041fa88ef693" is_pre_election: true
16:46:26.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.830725 30889 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 1869e0a1407440e79e0b04ecb71da9b0 in term 1.
16:46:26.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.832036 30753 leader_election.cc:304] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 083204b027784cb697b1041fa88ef693, 1869e0a1407440e79e0b04ecb71da9b0; no voters: b76f6b3da73b432fb8397d6dd53510c1
16:46:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.833801 30944 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Leader pre-election won for term 2
16:46:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.834160 30944 raft_consensus.cc:493] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
16:46:26.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.834563 30944 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 1 FOLLOWER]: Advancing to term 2
16:46:26.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.846983 30944 raft_consensus.cc:515] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:26.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.850489 30889 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "1869e0a1407440e79e0b04ecb71da9b0" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "083204b027784cb697b1041fa88ef693"
16:46:26.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.851045 30889 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 1 FOLLOWER]: Advancing to term 2
16:46:26.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.865924 30753 leader_election.cc:336] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer b76f6b3da73b432fb8397d6dd53510c1 (127.29.53.60:38873): Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111)
16:46:26.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.868283 30889 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1869e0a1407440e79e0b04ecb71da9b0 in term 2.
16:46:26.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.869442 30753 leader_election.cc:304] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 083204b027784cb697b1041fa88ef693, 1869e0a1407440e79e0b04ecb71da9b0; no voters: b76f6b3da73b432fb8397d6dd53510c1
16:46:26.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.871034 30944 leader_election.cc:290] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [CANDIDATE]: Term 2 election: Requested vote from peers 083204b027784cb697b1041fa88ef693 (127.29.53.61:44169), b76f6b3da73b432fb8397d6dd53510c1 (127.29.53.60:38873)
16:46:26.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.871654 30944 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 2 FOLLOWER]: Leader election won for term 2
16:46:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.872143 30944 raft_consensus.cc:697] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [term 2 LEADER]: Becoming Leader. State: Replica: 1869e0a1407440e79e0b04ecb71da9b0, State: Running, Role: LEADER
16:46:26.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.873095 30944 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:26.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.898025 30948 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1869e0a1407440e79e0b04ecb71da9b0. Latest consensus state: current_term: 2 leader_uuid: "1869e0a1407440e79e0b04ecb71da9b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:26.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.898916 30948 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: This master's current role is: LEADER
16:46:26.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.902966 30915 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
16:46:26.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.903316 30915 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
16:46:26.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.903847 30915 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
16:46:26.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.904222 30915 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
16:46:26.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.904525 30915 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
16:46:26.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.904805 30915 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
16:46:26.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:26.905028 30915 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38345/jwks.json
16:46:26.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.914616 30950 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
16:46:26.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.933535 30950 catalog_manager.cc:1494] Initializing Kudu cluster ID...
16:46:26.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.947165 30950 catalog_manager.cc:1269] Loaded cluster ID: 524b096ae3d14ebea944ac34909d7ad6
16:46:26.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.947674 30950 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
16:46:26.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.963431 30950 catalog_manager.cc:1514] Loading token signing keys...
16:46:26.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.968760 30950 catalog_manager.cc:6033] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0: Loaded TSK: 0
16:46:26.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:26.970418 30950 catalog_manager.cc:1524] Initializing in-progress tserver states...
16:46:27.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.026420 30915 master_runner.cc:386] Master server non-default flags:
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.53.20:40303
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/krb5kdc/kudu.keytab
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.29.53.60
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.53.60:38873
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38345/jwks.json
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
16:46:27.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data/info.pb
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.53.60
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46741
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/logs
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 60f5e5267b92c39485a66121d3ce3cc7ef57b0e0
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 29 Oct 2025 16:33:57 UTC on 5fd53c4cbb9d
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 8777
16:46:27.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
16:46:27.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.030059 30915 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
16:46:27.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.031728 30915 file_cache.cc:492] Constructed file cache file cache with capacity 419430
16:46:27.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.046834 30879 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "17db4142df3a468384b9d16f6c2bd85a" instance_seqno: 1761756375566216) as {username='kudu', principal='kudu/127.29.53.2@KRBTEST.COM'} at 127.29.53.2:56499; Asking this server to re-register.
16:46:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.048513 30546 heartbeater.cc:461] Registering TS with master...
16:46:27.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.051586 30879 ts_manager.cc:194] Registered new tserver with Master: 17db4142df3a468384b9d16f6c2bd85a (127.29.53.2:38379)
16:46:27.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.073437 30915 server_base.cc:1047] running on GCE node
16:46:27.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.367208 30889 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [term 2 FOLLOWER]: Refusing update from remote peer 1869e0a1407440e79e0b04ecb71da9b0: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
16:46:27.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.369726 30948 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
16:46:27.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.409700 30962 sys_catalog.cc:455] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1869e0a1407440e79e0b04ecb71da9b0. Latest consensus state: current_term: 2 leader_uuid: "1869e0a1407440e79e0b04ecb71da9b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:27.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.410672 30962 sys_catalog.cc:458] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: This master's current role is: FOLLOWER
16:46:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.429865 30962 sys_catalog.cc:455] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "1869e0a1407440e79e0b04ecb71da9b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:27.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.430711 30962 sys_catalog.cc:458] T 00000000000000000000000000000000 P 083204b027784cb697b1041fa88ef693 [sys.catalog]: This master's current role is: FOLLOWER
16:46:27.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.437325 30964 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "1869e0a1407440e79e0b04ecb71da9b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:27.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:27.442390 30964 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 [sys.catalog]: This master's current role is: LEADER
16:46:27.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:27.450603 30753 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 -> Peer b76f6b3da73b432fb8397d6dd53510c1 (127.29.53.60:38873): Couldn't send request to peer b76f6b3da73b432fb8397d6dd53510c1. Status: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
16:46:27.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:27.518141 30541 debug-util.cc:398] Leaking SignalData structure 0x7b08000b95e0 after lost signal to thread 30409
16:46:27.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:27.519764 30541 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba920 after lost signal to thread 30544
16:46:28.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:28.551797 30397 heartbeater.cc:499] Master 127.29.53.62:41263 was elected leader, sending a full tablet report...
16:46:28.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:28.573531 30716 master_runner.cc:186] Error getting master registration for 127.29.53.60:38873: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111),
16:46:28.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:28.574100 30716 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
16:46:28.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:28.622479 30706 heartbeater.cc:499] Master 127.29.53.62:41263 was elected leader, sending a full tablet report...
16:46:28.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:28.923214 30548 heartbeater.cc:499] Master 127.29.53.62:41263 was elected leader, sending a full tablet report...
16:46:29.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:29 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756389, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.602595 30582 init.cc:295] Successfully reacquired a new kerberos TGT
16:46:29.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:27.074919 30957 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:27.074811 30955 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:29.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:28.074263 30915 thread.cc:527] Waited for 1000ms trying to join with AWS (tid 30954)
16:46:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:27.073832 30954 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
16:46:29.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.757902 30915 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
16:46:29.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.781049 30915 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
16:46:29.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.788864 30915 hybrid_clock.cc:648] HybridClock initialized: now 1761756389788759 us; error 111 us; skew 500 ppm
16:46:29.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.790131 30915 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
16:46:29.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:29 dist-test-slave-wkbt krb5kdc[29920](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756389, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
16:46:29.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.804611 30915 init.cc:377] Logged in from keytab as kudu/127.29.53.60@KRBTEST.COM (short username kudu)
16:46:29.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.831583 30915 webserver.cc:492] Webserver started at http://127.29.53.60:46741/ using document root <none> and password file <none>
16:46:29.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.833354 30915 fs_manager.cc:362] Metadata directory not provided
16:46:29.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.833833 30915 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
16:46:29.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.852442 30915 fs_manager.cc:714] Time spent opening directory manager: real 0.013s user 0.010s sys 0.001s
16:46:29.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.864706 30989 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
16:46:29.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.866293 30915 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.005s sys 0.004s
16:46:29.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.866695 30915 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data,/tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal
16:46:29.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b76f6b3da73b432fb8397d6dd53510c1"
16:46:29.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-10-29 16:46:07 on dist-test-slave-wkbt"
16:46:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:29.873622 30915 fs_report.cc:389] FS layout report
16:46:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
16:46:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal
16:46:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/wal
16:46:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data/data
16:46:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
16:46:29.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
16:46:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
16:46:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
16:46:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
16:46:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
16:46:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
16:46:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
16:46:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
16:46:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
16:46:29.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
16:46:29.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:29.989774 30753 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111) [suppressed 8 similar messages]
16:46:29.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:29.995400 30753 consensus_peers.cc:597] T 00000000000000000000000000000000 P 1869e0a1407440e79e0b04ecb71da9b0 -> Peer b76f6b3da73b432fb8397d6dd53510c1 (127.29.53.60:38873): Couldn't send request to peer b76f6b3da73b432fb8397d6dd53510c1. Status: Network error: Client connection negotiation failed: client connection to 127.29.53.60:38873: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
16:46:30.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.009965 30915 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
16:46:30.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.018291 30915 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
16:46:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.018796 30915 kserver.cc:163] Server-wide thread pool size limit: 3276
16:46:30.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.258845 30915 rpc_server.cc:307] RPC server started. Bound to: 127.29.53.60:38873
16:46:30.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.259105 31042 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.53.60:38873 every 8 connection(s)
16:46:30.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.264017 30915 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskXsbxVh/test-tmp/mini-kudu-cluster6438405350542120599/master-2/data/info.pb
16:46:30.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.269331 29908 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu as pid 30915
16:46:30.271 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
16:46:30.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.331400 31043 sys_catalog.cc:263] Verifying existing consensus state
16:46:30.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.348225 31043 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1: Bootstrap starting.
16:46:30.365 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.53.62:41263(127.29.53.62:41263)] connecting to peer
16:46:30.445 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.53.61:44169(127.29.53.61:44169)] connecting to peer
16:46:30.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.449832 31043 log.cc:826] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1: Log is configured to *not* fsync() on all Append() calls
16:46:30.451 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.29.53.60:38873(127.29.53.60:38873)] connecting to peer
16:46:30.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.476964 31043 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
16:46:30.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.478652 31043 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1: Bootstrap complete.
16:46:30.501 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.29.53.61:44169(127.29.53.61:44169)] Successfully connected to peer
16:46:30.501 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.29.53.62:41263(127.29.53.62:41263)] Successfully connected to peer
16:46:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.550051 31043 raft_consensus.cc:359] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.555006 31043 raft_consensus.cc:740] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b76f6b3da73b432fb8397d6dd53510c1, State: Initialized, Role: FOLLOWER
16:46:30.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.556083 31043 consensus_queue.cc:260] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } }
16:46:30.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.572525 31052 sys_catalog.cc:455] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:30.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.573750 31052 sys_catalog.cc:458] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: This master's current role is: FOLLOWER
16:46:30.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.578289 31043 sys_catalog.cc:565] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: configured and running, proceeding with master startup.
16:46:30.598 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.29.53.60:38873(127.29.53.60:38873)] Successfully connected to peer
16:46:30.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:30 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756383, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.62@KRBTEST.COM for kudu/127.29.53.60@KRBTEST.COM
16:46:30.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.635012 31018 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [term 1 FOLLOWER]: Advancing to term 2
16:46:30.637 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
16:46:30.640 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.53.62:41263(127.29.53.62:41263)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:46:30.642 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.53.62:41263: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:46:30.643 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.29.53.61:44169(127.29.53.61:44169)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:46:30.645 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.29.53.61:44169: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:46:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:30.645471 31050 negotiation.cc:337] Failed RPC negotiation. Trace:
16:46:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.493927 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:39178 (local address 127.29.53.62:41263)
16:46:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.528201 (+ 34274us) server_negotiation.cc:207] Beginning negotiation
16:46:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.528231 (+ 30us) server_negotiation.cc:400] Waiting for connection header
16:46:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.550148 (+ 21917us) server_negotiation.cc:408] Connection header received
16:46:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.598343 (+ 48195us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:46:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.598379 (+ 36us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:46:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.598791 (+ 412us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:46:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.599410 (+ 619us) server_negotiation.cc:227] Negotiated authn=SASL
16:46:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.643828 (+ 44418us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:39178: BlockingRecv error: recv got EOF from 127.0.0.1:39178 (error 108)
16:46:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":33626,"thread_start_us":34253,"threads_started":1}
16:46:30.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:30.648873 31049 negotiation.cc:337] Failed RPC negotiation. Trace:
16:46:30.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.490001 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37044 (local address 127.29.53.61:44169)
16:46:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.520913 (+ 30912us) server_negotiation.cc:207] Beginning negotiation
16:46:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.520946 (+ 33us) server_negotiation.cc:400] Waiting for connection header
16:46:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.549647 (+ 28701us) server_negotiation.cc:408] Connection header received
16:46:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.600153 (+ 50506us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:46:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.600215 (+ 62us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:46:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.600667 (+ 452us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:46:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.600950 (+ 283us) server_negotiation.cc:227] Negotiated authn=SASL
16:46:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.646745 (+ 45795us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37044: BlockingRecv error: recv got EOF from 127.0.0.1:37044 (error 108)
16:46:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":30304,"thread_start_us":35834,"threads_started":1}
16:46:30.652 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.29.53.61:44169(127.29.53.61:44169)] handling channelInactive
16:46:30.650 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.29.53.60:38873(127.29.53.60:38873)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:46:30.653 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.29.53.60:38873: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
16:46:30.661 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=383), Traces: [0ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.53.62:41263, [158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.53.61:44169, [165ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.53.60:38873, [355ms] Sub RPC ConnectToMaster: received response from server master-127.29.53.62:41263: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [357ms] Sub RPC ConnectToMaster: received response from server master-127.29.53.61:44169: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [366ms] Sub RPC ConnectToMaster: received response from server master-127.29.53.60:38873: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
16:46:30.669 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=391), Traces: [0ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.53.62:41263, [158ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.53.61:44169, [165ms] Sub RPC ConnectToMaster: sending RPC to server master-127.29.53.60:38873, [355ms] Sub RPC ConnectToMaster: received response from server master-127.29.53.62:41263: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [357ms] Sub RPC ConnectToMaster: received response from server master-127.29.53.61:44169: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [366ms] Sub RPC ConnectToMaster: received response from server master-127.29.53.60:38873: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
16:46:30.672 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.29.53.62:41263(127.29.53.62:41263)] disconnecting while in state TERMINATED
16:46:30.673 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.29.53.60:38873(127.29.53.60:38873)] disconnecting while in state TERMINATED
16:46:30.673 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.29.53.61:44169(127.29.53.61:44169)] disconnecting while in state TERMINATED
16:46:30.687 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.29.53.62:41263(127.29.53.62:41263)] handling channelInactive
16:46:30.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:30.689273 31051 negotiation.cc:337] Failed RPC negotiation. Trace:
16:46:30.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.500308 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:38762 (local address 127.29.53.60:38873)
16:46:30.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.526924 (+ 26616us) server_negotiation.cc:207] Beginning negotiation
16:46:30.689 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:46:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.526981 (+ 57us) server_negotiation.cc:400] Waiting for connection header
16:46:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.599190 (+ 72209us) server_negotiation.cc:408] Connection header received
16:46:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.608544 (+ 9354us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
16:46:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.608578 (+ 34us) server_negotiation.cc:462] Received NEGOTIATE request from client
16:46:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.608845 (+ 267us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
16:46:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.609453 (+ 608us) server_negotiation.cc:227] Negotiated authn=SASL
16:46:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1029 16:46:30.687084 (+ 77631us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:38762: BlockingRecv error: recv got EOF from 127.0.0.1:38762 (error 108)
16:46:30.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":23203,"thread_start_us":22592,"threads_started":1}
16:46:30.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:30 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756382, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.1@KRBTEST.COM for kudu/127.29.53.60@KRBTEST.COM
16:46:30.700 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.29.53.60:38873(127.29.53.60:38873)] handling channelInactive
16:46:30.701 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:46:30.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.737509 29908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 30256
16:46:30.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:30 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756389, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.3@KRBTEST.COM for kudu/127.29.53.60@KRBTEST.COM
16:46:30.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.785494 31052 sys_catalog.cc:455] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1869e0a1407440e79e0b04ecb71da9b0. Latest consensus state: current_term: 2 leader_uuid: "1869e0a1407440e79e0b04ecb71da9b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:30.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.786371 31052 sys_catalog.cc:458] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: This master's current role is: FOLLOWER
16:46:30.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.788142 31072 sys_catalog.cc:455] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "1869e0a1407440e79e0b04ecb71da9b0" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "1869e0a1407440e79e0b04ecb71da9b0" member_type: VOTER last_known_addr { host: "127.29.53.62" port: 41263 } } peers { permanent_uuid: "083204b027784cb697b1041fa88ef693" member_type: VOTER last_known_addr { host: "127.29.53.61" port: 44169 } } peers { permanent_uuid: "b76f6b3da73b432fb8397d6dd53510c1" member_type: VOTER last_known_addr { host: "127.29.53.60" port: 38873 } } }
16:46:30.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.788925 31072 sys_catalog.cc:458] T 00000000000000000000000000000000 P b76f6b3da73b432fb8397d6dd53510c1 [sys.catalog]: This master's current role is: FOLLOWER
16:46:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.803233 30701 heartbeater.cc:344] Connected to a master server at 127.29.53.60:38873
16:46:30.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.803732 30701 heartbeater.cc:499] Master 127.29.53.60:38873 was elected leader, sending a full tablet report...
16:46:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.800899 31076 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
16:46:30.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.822816 31008 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "28fceda62ce942b68bab0f8c70c336e4" instance_seqno: 1761756380129724) as {username='kudu', principal='kudu/127.29.53.3@KRBTEST.COM'} at 127.29.53.3:54393; Asking this server to re-register.
16:46:30.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.826380 30701 heartbeater.cc:461] Registering TS with master...
16:46:30.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.831525 31008 ts_manager.cc:194] Registered new tserver with Master: 28fceda62ce942b68bab0f8c70c336e4 (127.29.53.3:43235)
16:46:30.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.839986 29908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 30407
16:46:30.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:30 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756389, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.60@KRBTEST.COM for kudu/127.29.53.62@KRBTEST.COM
16:46:30.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:30.842233 30991 connection.cc:537] server connection from 127.29.53.1:32949 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
16:46:30.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:30 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756389, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.60@KRBTEST.COM for kudu/127.29.53.61@KRBTEST.COM
16:46:30.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.871968 29908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 30564
16:46:30.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Oct 29 16:46:30 dist-test-slave-wkbt krb5kdc[29920](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1761756389, etypes {rep=17 tkt=17 ses=17}, kudu/127.29.53.60@KRBTEST.COM for kudu/127.29.53.60@KRBTEST.COM
16:46:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.912355 29908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 30716
16:46:30.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.959079 29908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 30803
16:46:30.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:30.989589 29908 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 30915
16:46:31.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T16:46:31Z chronyd exiting
16:46:32.131 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
16:46:32.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:32.132874 29176 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 29458
16:46:32.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:32.157935 29176 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 29618
16:46:32.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:32.188238 29176 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 29763
16:46:32.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:32.214704 29176 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 29188
16:46:32.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:32.244589 29176 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 29258
16:46:32.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:32.275054 29363 connection.cc:537] client connection to 127.28.126.61:45025 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
16:46:32.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251029 16:46:32.275647 29363 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107) [suppressed 1 similar messages]
16:46:32.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251029 16:46:32.276111 29176 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskXsbxVh/build/tsan/bin/kudu with pid 29333
16:46:33.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-10-29T16:46:33Z chronyd exiting
16:46:34.457 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.29.53.62:41263,127.29.53.61:44169,127.29.53.60:38873). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more