21:06:47.481 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:06:47.493 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskGYhehh/test-tmp
21:06:47.497 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:06:47.500 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskGYhehh/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:06:47.828 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:06:47.964 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:06:47.965 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:06:47.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:47.969461 31282 mini_oidc.cc:150] Starting JWKS server
21:06:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:47.973500 31282 webserver.cc:492] Webserver started at http://127.0.0.1:34467/ using document root <none> and password file <none>
21:06:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:47.973959 31282 mini_oidc.cc:186] Starting OIDC Discovery server
21:06:47.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:47.975960 31282 webserver.cc:492] Webserver started at http://127.0.0.1:44233/ using document root <none> and password file <none>
21:06:48.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-12T21:06:48Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:06:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-12T21:06:48Z Disabled control of system clock
21:06:48.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:48.040015 31282 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:06:48.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:06:48.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/wal
21:06:48.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/data
21:06:48.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:48.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:48.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:48.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:48.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:48.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:48.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/logs
21:06:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/data/info.pb
21:06:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:48.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:48.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:48.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:48.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:48.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:48.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:06:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.190:40487
21:06:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.190
21:06:48.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:06:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:06:48.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:48.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34467/jwks.json with env {}
21:06:48.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:48.358197 31294 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:48.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:48.358891 31294 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:48.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:48.359512 31294 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:48.392078 31294 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:06:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:48.392390 31294 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:48.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:48.392596 31294 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:06:48.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:48.392788 31294 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:06:48.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:48.393231 31294 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:06:48.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:48.393438 31294 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34467/jwks.json
21:06:48.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:48.430478 31294 master_runner.cc:386] Master server non-default flags:
21:06:48.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:48.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:06:48.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:48.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:48.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/data
21:06:48.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/wal
21:06:48.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:48.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:06:48.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:48.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:48.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:48.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.190:40487
21:06:48.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:48.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:48.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:48.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34467/jwks.json
21:06:48.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:48.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/data/info.pb
21:06:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.190
21:06:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:48.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/logs
21:06:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:06:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:06:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:48.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:06:48.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:06:48.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:48.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:48.432168 31294 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:48.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:48.434521 31294 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:48.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:48.447433 31300 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:49.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:48.447521 31301 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:49.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:49.669289 31302 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1218 milliseconds
21:06:49.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:49.670583 31303 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:49.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:49.673352 31294 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.226s user 0.356s sys 0.869s
21:06:49.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:49.673653 31294 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.227s user 0.356s sys 0.869s
21:06:49.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.673960 31294 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:06:49.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.681619 31294 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:49.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.684582 31294 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:49.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.686093 31294 hybrid_clock.cc:648] HybridClock initialized: now 1765573609686037 us; error 147 us; skew 500 ppm
21:06:49.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.687115 31294 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:49.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.695087 31294 webserver.cc:492] Webserver started at http://127.30.140.190:43141/ using document root <none> and password file <none>
21:06:49.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.696267 31294 fs_manager.cc:362] Metadata directory not provided
21:06:49.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.696507 31294 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:49.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.697144 31294 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:49.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.702109 31294 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/data/instance:
21:06:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e"
21:06:49.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:49 on dist-test-slave-vhkf"
21:06:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.703789 31294 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/wal/instance:
21:06:49.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e"
21:06:49.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:49 on dist-test-slave-vhkf"
21:06:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.713335 31294 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
21:06:49.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.720831 31310 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.722362 31294 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.003s
21:06:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.722764 31294 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/wal
21:06:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e"
21:06:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:49 on dist-test-slave-vhkf"
21:06:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.723223 31294 fs_report.cc:389] FS layout report
21:06:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/wal
21:06:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/wal
21:06:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/data/data
21:06:49.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:49.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:49.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:49.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:49.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:49.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:49.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:49.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:49.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:49.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:49.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:49.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.795797 31294 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:49.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.797699 31294 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:49.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.798239 31294 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:49.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.881634 31294 rpc_server.cc:307] RPC server started. Bound to: 127.30.140.190:40487
21:06:49.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.881772 31361 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.140.190:40487 every 8 connection(s)
21:06:49.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.884946 31294 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/data/info.pb
21:06:49.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.891907 31362 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:06:49.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.894279 31282 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 31294
21:06:49.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.894879 31282 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-0/wal/instance
21:06:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.900579 31282 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:06:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:06:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/wal
21:06:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/data
21:06:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:49.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/logs
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/data/info.pb
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:06:49.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:49.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:49.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:49.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.189:40703
21:06:49.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.189
21:06:49.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:49.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:06:49.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:49.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:49.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:49.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:49.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:06:49.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:49.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34467/jwks.json with env {}
21:06:49.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.909932 31362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } has no permanent_uuid. Determining permanent_uuid...
21:06:49.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:49.943866 31362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } has no permanent_uuid. Determining permanent_uuid...
21:06:49.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:49.946681 31311 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.140.189:40703: connect: Connection refused (error 111)
21:06:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:49.951552 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.189:40703: Network error: Client connection negotiation failed: client connection to 127.30.140.189:40703: connect: Connection refused (error 111)
21:06:50.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:50.001183 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } attempt: 1
21:06:50.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.007510 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.189:40703: Network error: Client connection negotiation failed: client connection to 127.30.140.189:40703: connect: Connection refused (error 111)
21:06:50.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:50.075976 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } attempt: 2
21:06:50.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.080972 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.189:40703: Network error: Client connection negotiation failed: client connection to 127.30.140.189:40703: connect: Connection refused (error 111)
21:06:50.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:50.172412 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } attempt: 3
21:06:50.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.176888 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.189:40703: Network error: Client connection negotiation failed: client connection to 127.30.140.189:40703: connect: Connection refused (error 111)
21:06:50.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.243125 31364 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:50.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.243865 31364 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:50.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.244572 31364 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:50.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.277493 31364 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:06:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.277892 31364 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.278219 31364 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:06:50.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.278565 31364 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:06:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.278913 31364 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:06:50.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.279158 31364 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34467/jwks.json
21:06:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:50.315793 31364 master_runner.cc:386] Master server non-default flags:
21:06:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:06:50.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/data
21:06:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/wal
21:06:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:06:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:50.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.189:40703
21:06:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34467/jwks.json
21:06:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/data/info.pb
21:06:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.189
21:06:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:50.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:50.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:50.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/logs
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:50.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:50.317956 31364 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:50.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:50.320474 31364 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:50.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:50.320336 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } attempt: 4
21:06:50.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.324870 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.189:40703: Network error: Client connection negotiation failed: client connection to 127.30.140.189:40703: connect: Connection refused (error 111)
21:06:50.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.332616 31373 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:50.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:50.624373 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } attempt: 5
21:06:50.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.633776 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.189:40703: Network error: Client connection negotiation failed: client connection to 127.30.140.189:40703: connect: Connection refused (error 111)
21:06:51.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.181432 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } attempt: 6
21:06:51.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:51.196024 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.189:40703: Network error: Client connection negotiation failed: client connection to 127.30.140.189:40703: connect: Connection refused (error 111)
21:06:51.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:50.334553 31374 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:51.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:51.593343 31376 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:51.597523 31375 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1257 milliseconds
21:06:51.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.597673 31364 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:06:51.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.599403 31364 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:51.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.603320 31364 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.604799 31364 hybrid_clock.cc:648] HybridClock initialized: now 1765573611604758 us; error 67 us; skew 500 ppm
21:06:51.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.605736 31364 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:51.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.613082 31364 webserver.cc:492] Webserver started at http://127.30.140.189:36573/ using document root <none> and password file <none>
21:06:51.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.614122 31364 fs_manager.cc:362] Metadata directory not provided
21:06:51.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.614331 31364 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:51.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.614857 31364 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:51.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.619800 31364 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/data/instance:
21:06:51.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a36f768c5484b83847fe711afa658d9"
21:06:51.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:51 on dist-test-slave-vhkf"
21:06:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.621049 31364 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/wal/instance:
21:06:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a36f768c5484b83847fe711afa658d9"
21:06:51.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:51 on dist-test-slave-vhkf"
21:06:51.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.628899 31364 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.000s
21:06:51.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.635178 31385 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:51.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.636345 31364 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
21:06:51.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.636732 31364 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/wal
21:06:51.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a36f768c5484b83847fe711afa658d9"
21:06:51.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:51 on dist-test-slave-vhkf"
21:06:51.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.637176 31364 fs_report.cc:389] FS layout report
21:06:51.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:51.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/wal
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/wal
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/data/data
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:51.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:51.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.685992 31364 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:51.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.687724 31364 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:51.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.688266 31364 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:51.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.770183 31364 rpc_server.cc:307] RPC server started. Bound to: 127.30.140.189:40703
21:06:51.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.770237 31436 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.140.189:40703 every 8 connection(s)
21:06:51.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.773447 31364 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/data/info.pb
21:06:51.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.776988 31282 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 31364
21:06:51.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.777671 31282 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-1/wal/instance
21:06:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.779906 31282 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:06:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:06:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/wal
21:06:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/data
21:06:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:51.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/logs
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/data/info.pb
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:06:51.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.188:44343
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.188
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34467/jwks.json with env {}
21:06:51.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.781797 31437 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:06:51.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.795272 31437 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } has no permanent_uuid. Determining permanent_uuid...
21:06:51.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.819118 31437 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } has no permanent_uuid. Determining permanent_uuid...
21:06:51.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.836020 31437 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } has no permanent_uuid. Determining permanent_uuid...
21:06:51.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:51.839221 31387 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:51.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:51.844211 31437 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.893774 31437 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 1
21:06:51.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:51.899315 31437 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:51.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:51.967911 31437 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 2
21:06:51.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:51.972632 31437 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:52.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.064112 31437 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 3
21:06:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.068488 31437 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.124749 31439 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:52.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.125445 31439 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:52.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.126194 31439 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.160140 31439 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:06:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.160460 31439 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.160657 31439 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:06:52.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.160845 31439 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:06:52.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.161168 31439 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:06:52.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.161405 31439 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:34467/jwks.json
21:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.199682 31439 master_runner.cc:386] Master server non-default flags:
21:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/data
21:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/wal
21:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:52.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.188:44343
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:34467/jwks.json
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/data/info.pb
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.188
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:52.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:52.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/logs
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.201145 31439 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:52.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.203315 31439 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:52.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.211954 31437 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 4
21:06:52.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.219949 31437 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:52.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.220495 31450 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:52.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.256704 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } attempt: 7
21:06:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.272313 31362 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } has no permanent_uuid. Determining permanent_uuid...
21:06:52.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.280040 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:52.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.338510 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 1
21:06:52.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.343623 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:52.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.425077 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 2
21:06:52.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.430226 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.515715 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 3
21:06:52.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.519397 31437 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 5
21:06:52.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.521255 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:52.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.524909 31437 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:52.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.661701 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 4
21:06:52.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.667057 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:52.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:52.950513 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 5
21:06:52.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.956439 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:53.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.072384 31437 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 6
21:06:53.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:53.081966 31437 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.223948 31449 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:53.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:52.225363 31452 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:53.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:53.454725 31451 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:06:53.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.455075 31439 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:06:53.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.459702 31439 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:53.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.462816 31439 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:53.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.464326 31439 hybrid_clock.cc:648] HybridClock initialized: now 1765573613464259 us; error 70 us; skew 500 ppm
21:06:53.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.465432 31439 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:53.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.472877 31439 webserver.cc:492] Webserver started at http://127.30.140.188:40645/ using document root <none> and password file <none>
21:06:53.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.474121 31439 fs_manager.cc:362] Metadata directory not provided
21:06:53.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.474409 31439 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:53.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.474994 31439 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:53.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.480252 31439 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/data/instance:
21:06:53.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f117bff8629a4e43827fdde1fa75ce4d"
21:06:53.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:53 on dist-test-slave-vhkf"
21:06:53.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.481712 31439 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/wal/instance:
21:06:53.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f117bff8629a4e43827fdde1fa75ce4d"
21:06:53.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:53 on dist-test-slave-vhkf"
21:06:53.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.490374 31439 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.003s sys 0.005s
21:06:53.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.497339 31463 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:53.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.498687 31439 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.001s
21:06:53.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.499138 31439 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/wal
21:06:53.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f117bff8629a4e43827fdde1fa75ce4d"
21:06:53.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:53 on dist-test-slave-vhkf"
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.499576 31439 fs_report.cc:389] FS layout report
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/wal
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/wal
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/data/data
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:53.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:53.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.508982 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 6
21:06:53.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:53.517524 31362 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.140.188:44343: Network error: Client connection negotiation failed: client connection to 127.30.140.188:44343: connect: Connection refused (error 111)
21:06:53.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.562892 31439 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:53.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.564626 31439 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:53.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.565091 31439 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:53.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.649044 31439 rpc_server.cc:307] RPC server started. Bound to: 127.30.140.188:44343
21:06:53.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.649104 31516 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.140.188:44343 every 8 connection(s)
21:06:53.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.653028 31439 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/data/info.pb
21:06:53.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.654522 31282 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 31439
21:06:53.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.655171 31282 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/master-2/wal/instance
21:06:53.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.661648 31517 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:06:53.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.678716 31517 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } has no permanent_uuid. Determining permanent_uuid...
21:06:53.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.705447 31517 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } has no permanent_uuid. Determining permanent_uuid...
21:06:53.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.716024 31517 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } has no permanent_uuid. Determining permanent_uuid...
21:06:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.734139 31517 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d: Bootstrap starting.
21:06:53.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.742062 31517 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d: Neither blocks nor log segments found. Creating new log.
21:06:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.744127 31517 log.cc:826] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d: Log is configured to *not* fsync() on all Append() calls
21:06:53.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.750473 31517 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d: No bootstrap required, opened a new log
21:06:53.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.770313 31517 raft_consensus.cc:359] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } }
21:06:53.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.771880 31517 raft_consensus.cc:385] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:06:53.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.772598 31517 raft_consensus.cc:740] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f117bff8629a4e43827fdde1fa75ce4d, State: Initialized, Role: FOLLOWER
21:06:53.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.774039 31517 consensus_queue.cc:260] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } }
21:06:53.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.777441 31525 sys_catalog.cc:455] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } } }
21:06:53.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.778927 31525 sys_catalog.cc:458] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [sys.catalog]: This master's current role is: FOLLOWER
21:06:53.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.779253 31517 sys_catalog.cc:565] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [sys.catalog]: configured and running, proceeding with master startup.
21:06:53.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:53.812108 31536 catalog_manager.cc:1568] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:06:53.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:53.812433 31536 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:06:53.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:53.820505 31471 tablet.cc:2378] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:06:53.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.808565 31439 master_runner.cc:186] Error getting master registration for 127.30.140.190:40487: OK, instance_id { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" instance_seqno: 1765573609857336 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:06:53.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:53.837034 31439 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:06:54.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.020701 31525 raft_consensus.cc:493] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:06:54.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.021278 31525 raft_consensus.cc:515] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } }
21:06:54.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.023626 31525 leader_election.cc:290] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7fba6fd8e0ad4260b0fdbff15fbe989e (127.30.140.190:40487), 2a36f768c5484b83847fe711afa658d9 (127.30.140.189:40703)
21:06:54.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.142392 31437 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 7
21:06:54.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.169176 31437 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9: Bootstrap starting.
21:06:54.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.178318 31437 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9: Neither blocks nor log segments found. Creating new log.
21:06:54.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.181284 31437 log.cc:826] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9: Log is configured to *not* fsync() on all Append() calls
21:06:54.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.188122 31437 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9: No bootstrap required, opened a new log
21:06:54.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.218894 31437 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } }
21:06:54.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.219913 31437 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:06:54.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.220245 31437 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a36f768c5484b83847fe711afa658d9, State: Initialized, Role: FOLLOWER
21:06:54.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.221338 31437 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } }
21:06:54.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.224282 31540 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } } }
21:06:54.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.226115 31540 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.227466 31437 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [sys.catalog]: configured and running, proceeding with master startup.
21:06:54.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.237923 31412 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f117bff8629a4e43827fdde1fa75ce4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a36f768c5484b83847fe711afa658d9" is_pre_election: true
21:06:54.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.238952 31412 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f117bff8629a4e43827fdde1fa75ce4d in term 0.
21:06:54.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.240288 31466 leader_election.cc:304] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a36f768c5484b83847fe711afa658d9, f117bff8629a4e43827fdde1fa75ce4d; no voters:
21:06:54.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.241477 31525 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [term 0 FOLLOWER]: Leader pre-election won for term 1
21:06:54.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.241941 31525 raft_consensus.cc:493] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:06:54.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.242288 31525 raft_consensus.cc:3060] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [term 0 FOLLOWER]: Advancing to term 1
21:06:54.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.248747 31525 raft_consensus.cc:515] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } }
21:06:54.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.252401 31412 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f117bff8629a4e43827fdde1fa75ce4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a36f768c5484b83847fe711afa658d9"
21:06:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.253417 31412 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [term 0 FOLLOWER]: Advancing to term 1
21:06:54.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.251022 31525 leader_election.cc:290] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [CANDIDATE]: Term 1 election: Requested vote from peers 7fba6fd8e0ad4260b0fdbff15fbe989e (127.30.140.190:40487), 2a36f768c5484b83847fe711afa658d9 (127.30.140.189:40703)
21:06:54.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.273444 31412 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f117bff8629a4e43827fdde1fa75ce4d in term 1.
21:06:54.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.274823 31466 leader_election.cc:304] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a36f768c5484b83847fe711afa658d9, f117bff8629a4e43827fdde1fa75ce4d; no voters:
21:06:54.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.275749 31525 raft_consensus.cc:2804] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [term 1 FOLLOWER]: Leader election won for term 1
21:06:54.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.278295 31525 raft_consensus.cc:697] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [term 1 LEADER]: Becoming Leader. State: Replica: f117bff8629a4e43827fdde1fa75ce4d, State: Running, Role: LEADER
21:06:54.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.279803 31525 consensus_queue.cc:237] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } }
21:06:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.296466 31550 sys_catalog.cc:455] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [sys.catalog]: SysCatalogTable state changed. Reason: New leader f117bff8629a4e43827fdde1fa75ce4d. Latest consensus state: current_term: 1 leader_uuid: "f117bff8629a4e43827fdde1fa75ce4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } } }
21:06:54.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.297358 31550 sys_catalog.cc:458] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [sys.catalog]: This master's current role is: LEADER
21:06:54.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.317526 31554 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:06:54.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.294351 31364 master_runner.cc:186] Error getting master registration for 127.30.140.190:40487: OK, instance_id { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" instance_seqno: 1765573609857336 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:06:54.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.324844 31364 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:06:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.326143 31554 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:06:54.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:54.334475 31553 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:06:54.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:54.335450 31553 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:06:54.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.353040 31412 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [term 1 FOLLOWER]: Refusing update from remote peer f117bff8629a4e43827fdde1fa75ce4d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:06:54.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.354894 31525 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [LEADER]: Connected to new peer: Peer: permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:06:54.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.388561 31540 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f117bff8629a4e43827fdde1fa75ce4d. Latest consensus state: current_term: 1 leader_uuid: "f117bff8629a4e43827fdde1fa75ce4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } } }
21:06:54.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.389750 31540 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.400344 31556 mvcc.cc:204] Tried to move back new op lower bound from 7231789524359831552 to 7231789524103372800. Current Snapshot: MvccSnapshot[applied={T|T < 7231789524359831552}]
21:06:54.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.403105 31525 sys_catalog.cc:455] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f117bff8629a4e43827fdde1fa75ce4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } } }
21:06:54.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.404551 31525 sys_catalog.cc:458] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [sys.catalog]: This master's current role is: LEADER
21:06:54.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.413700 31540 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f117bff8629a4e43827fdde1fa75ce4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } } }
21:06:54.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.414619 31540 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9 [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.415282 31554 catalog_manager.cc:1357] Generated new cluster ID: 3dd68a70a01644f4971bf3bace9dfa6c
21:06:54.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.415623 31554 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:06:54.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.483932 31554 catalog_manager.cc:1380] Generated new certificate authority record
21:06:54.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.486925 31554 catalog_manager.cc:1514] Loading token signing keys...
21:06:54.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.520836 31554 catalog_manager.cc:6027] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d: Generated new TSK 0
21:06:54.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.522768 31554 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:06:54.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.550884 31362 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } attempt: 7
21:06:54.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.585935 31282 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:06:54.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:06:54.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/wal
21:06:54.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/data
21:06:54.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:54.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/logs
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/data/info.pb
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.129:0
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.140.129
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.129
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:06:54.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:06:54.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:54.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:54.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:06:54.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.586097 31362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e: Bootstrap starting.
21:06:54.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.595122 31362 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e: Neither blocks nor log segments found. Creating new log.
21:06:54.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.597877 31362 log.cc:826] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e: Log is configured to *not* fsync() on all Append() calls
21:06:54.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.605031 31362 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e: No bootstrap required, opened a new log
21:06:54.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.638567 31362 raft_consensus.cc:359] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } }
21:06:54.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.640575 31362 raft_consensus.cc:385] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:06:54.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.641281 31362 raft_consensus.cc:740] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7fba6fd8e0ad4260b0fdbff15fbe989e, State: Initialized, Role: FOLLOWER
21:06:54.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.643759 31362 consensus_queue.cc:260] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } }
21:06:54.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.672688 31336 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f117bff8629a4e43827fdde1fa75ce4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e"
21:06:54.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.672405 31362 sys_catalog.cc:565] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [sys.catalog]: configured and running, proceeding with master startup.
21:06:54.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.675282 31335 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [term 0 FOLLOWER]: Advancing to term 1
21:06:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.668715 31564 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } } }
21:06:54.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.687759 31564 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.692008 31337 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "f117bff8629a4e43827fdde1fa75ce4d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" is_pre_election: true
21:06:54.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.698033 31335 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [term 1 FOLLOWER]: Refusing update from remote peer f117bff8629a4e43827fdde1fa75ce4d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:06:54.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.700232 31525 consensus_queue.cc:1048] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [LEADER]: Connected to new peer: Peer: permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:06:54.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.813777 31564 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [sys.catalog]: SysCatalogTable state changed. Reason: New leader f117bff8629a4e43827fdde1fa75ce4d. Latest consensus state: current_term: 1 leader_uuid: "f117bff8629a4e43827fdde1fa75ce4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } } }
21:06:54.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.815155 31564 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.811053 31573 sys_catalog.cc:455] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "f117bff8629a4e43827fdde1fa75ce4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } } }
21:06:54.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.815927 31573 sys_catalog.cc:458] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.855123 31581 catalog_manager.cc:1269] Loaded cluster ID: 3dd68a70a01644f4971bf3bace9dfa6c
21:06:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.855872 31581 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e: loading cluster ID for follower catalog manager: success
21:06:54.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.865355 31581 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e: acquiring CA information for follower catalog manager: success
21:06:54.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:54.884613 31581 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 7fba6fd8e0ad4260b0fdbff15fbe989e: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:06:55.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:55.285277 31583 sys_catalog.cc:455] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "f117bff8629a4e43827fdde1fa75ce4d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7fba6fd8e0ad4260b0fdbff15fbe989e" member_type: VOTER last_known_addr { host: "127.30.140.190" port: 40487 } } peers { permanent_uuid: "2a36f768c5484b83847fe711afa658d9" member_type: VOTER last_known_addr { host: "127.30.140.189" port: 40703 } } peers { permanent_uuid: "f117bff8629a4e43827fdde1fa75ce4d" member_type: VOTER last_known_addr { host: "127.30.140.188" port: 44343 } } }
21:06:55.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:55.292022 31583 sys_catalog.cc:458] T 00000000000000000000000000000000 P f117bff8629a4e43827fdde1fa75ce4d [sys.catalog]: This master's current role is: LEADER
21:06:55.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:55.343964 31553 catalog_manager.cc:1269] Loaded cluster ID: 3dd68a70a01644f4971bf3bace9dfa6c
21:06:55.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:55.344319 31553 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9: loading cluster ID for follower catalog manager: success
21:06:55.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:55.352875 31553 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9: acquiring CA information for follower catalog manager: success
21:06:55.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:55.371623 31553 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 2a36f768c5484b83847fe711afa658d9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:06:55.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:55.396381 31563 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:55.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:55.397625 31563 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:55.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:55.398721 31563 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:55.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:55.463703 31563 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:55.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:55.471001 31563 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.140.129
21:06:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:55.544920 31563 tablet_server_runner.cc:78] Tablet server non-default flags:
21:06:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:06:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:55.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/data
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/wal
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.129:0
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/data/info.pb
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.129
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.140.129
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/logs
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:55.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:06:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:06:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:06:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:06:55.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:55.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:55.554257 31563 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:55.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:55.557317 31563 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:55.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:55.605170 31591 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:57.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:56.989133 31590 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 31563
21:06:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:57.614254 31590 kernel_stack_watchdog.cc:198] Thread 31563 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 376ms:
21:06:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:06:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:06:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:06:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:06:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:57.614367 31563 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.004s user 0.693s sys 1.087s
21:06:57.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:57.614804 31563 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.005s user 0.693s sys 1.087s
21:06:57.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:55.612576 31592 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:57.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:57.633276 31596 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:57.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.633394 31563 server_base.cc:1047] running on GCE node
21:06:57.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.634938 31563 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:57.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.637989 31563 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:57.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.639634 31563 hybrid_clock.cc:648] HybridClock initialized: now 1765573617639550 us; error 97 us; skew 500 ppm
21:06:57.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.641103 31563 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:57.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.669777 31563 webserver.cc:492] Webserver started at http://127.30.140.129:39035/ using document root <none> and password file <none>
21:06:57.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.671219 31563 fs_manager.cc:362] Metadata directory not provided
21:06:57.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.671510 31563 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:57.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.672096 31563 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:57.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.679128 31563 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/data/instance:
21:06:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "483c99ed4d3f4d298aafe7d89f1a5d63"
21:06:57.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:57 on dist-test-slave-vhkf"
21:06:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.680891 31563 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/wal/instance:
21:06:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "483c99ed4d3f4d298aafe7d89f1a5d63"
21:06:57.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:57 on dist-test-slave-vhkf"
21:06:57.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.691864 31563 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.004s sys 0.004s
21:06:57.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.700708 31601 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:57.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.702564 31563 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.006s sys 0.000s
21:06:57.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.703058 31563 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/wal
21:06:57.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "483c99ed4d3f4d298aafe7d89f1a5d63"
21:06:57.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:06:57 on dist-test-slave-vhkf"
21:06:57.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.703549 31563 fs_report.cc:389] FS layout report
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/wal
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/wal
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/data/data
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:57.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:57.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.778681 31563 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.781273 31563 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:57.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.781953 31563 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:57.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.802073 31563 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:06:57.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.811761 31563 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:06:57.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.812119 31563 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:06:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.812455 31563 ts_tablet_manager.cc:616] Registered 0 tablets
21:06:57.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:57.812757 31563 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.001s
21:06:58.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.056871 31563 rpc_server.cc:307] RPC server started. Bound to: 127.30.140.129:44733
21:06:58.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.056931 31714 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.140.129:44733 every 8 connection(s)
21:06:58.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.061043 31563 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/data/info.pb
21:06:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.064535 31282 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 31563
21:06:58.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.065758 31282 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-0/wal/instance
21:06:58.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.102916 31282 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:06:58.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:06:58.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/wal
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/data
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/logs
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/data/info.pb
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:58.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:06:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.130:0
21:06:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.140.130
21:06:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.130
21:06:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:06:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:06:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:58.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.125571 31716 heartbeater.cc:344] Connected to a master server at 127.30.140.189:40703
21:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.125633 31715 heartbeater.cc:344] Connected to a master server at 127.30.140.188:44343
21:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.126276 31716 heartbeater.cc:461] Registering TS with master...
21:06:58.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.126561 31715 heartbeater.cc:461] Registering TS with master...
21:06:58.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.127972 31716 heartbeater.cc:507] Master 127.30.140.189:40703 requested a full tablet report, sending...
21:06:58.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.129441 31719 heartbeater.cc:344] Connected to a master server at 127.30.140.190:40487
21:06:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.129858 31719 heartbeater.cc:461] Registering TS with master...
21:06:58.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.129808 31715 heartbeater.cc:507] Master 127.30.140.188:44343 requested a full tablet report, sending...
21:06:58.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.130921 31719 heartbeater.cc:507] Master 127.30.140.190:40487 requested a full tablet report, sending...
21:06:58.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.133380 31482 ts_manager.cc:194] Registered new tserver with Master: 483c99ed4d3f4d298aafe7d89f1a5d63 (127.30.140.129:44733)
21:06:58.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.134582 31326 ts_manager.cc:194] Registered new tserver with Master: 483c99ed4d3f4d298aafe7d89f1a5d63 (127.30.140.129:44733)
21:06:58.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.136308 31482 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.140.129:41783
21:06:58.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.138175 31402 ts_manager.cc:194] Registered new tserver with Master: 483c99ed4d3f4d298aafe7d89f1a5d63 (127.30.140.129:44733)
21:06:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:58.470523 31725 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:58.471067 31725 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:58.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:58.471556 31725 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:58.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:58.507308 31725 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:58.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:58.508227 31725 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.140.130
21:06:58.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.546267 31725 tablet_server_runner.cc:78] Tablet server non-default flags:
21:06:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:06:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:58.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/data
21:06:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/wal
21:06:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.130:0
21:06:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/data/info.pb
21:06:58.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.130
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.140.130
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/logs
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.547704 31725 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:58.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:58.549623 31725 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:58.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:58.567485 31732 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:59.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:06:59.142889 31715 heartbeater.cc:499] Master 127.30.140.188:44343 was elected leader, sending a full tablet report...
21:07:00.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:59.971129 31731 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 31725
21:07:00.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:06:58.569648 31733 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:00.339418 31731 kernel_stack_watchdog.cc:198] Thread 31725 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
21:07:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:07:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:07:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:07:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:07:00.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:00.340533 31725 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.772s user 0.579s sys 1.161s
21:07:00.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:00.341782 31725 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.774s user 0.579s sys 1.161s
21:07:00.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:00.343194 31736 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:00.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:00.347108 31734 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1773 milliseconds
21:07:00.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.347191 31725 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:00.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.348590 31725 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:00.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.351123 31725 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:00.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.352499 31725 hybrid_clock.cc:648] HybridClock initialized: now 1765573620352447 us; error 66 us; skew 500 ppm
21:07:00.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.353624 31725 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:00.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.361114 31725 webserver.cc:492] Webserver started at http://127.30.140.130:42455/ using document root <none> and password file <none>
21:07:00.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.362267 31725 fs_manager.cc:362] Metadata directory not provided
21:07:00.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.362521 31725 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:00.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.363190 31725 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:07:00.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.368017 31725 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/data/instance:
21:07:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "daf27ab253ce429ea73b8dc04b3f5b44"
21:07:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:00 on dist-test-slave-vhkf"
21:07:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.369575 31725 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/wal/instance:
21:07:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "daf27ab253ce429ea73b8dc04b3f5b44"
21:07:00.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:00 on dist-test-slave-vhkf"
21:07:00.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.377628 31725 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
21:07:00.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.384013 31746 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:00.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.385392 31725 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.007s sys 0.000s
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.385736 31725 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/wal
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "daf27ab253ce429ea73b8dc04b3f5b44"
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:00 on dist-test-slave-vhkf"
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.386073 31725 fs_report.cc:389] FS layout report
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/wal
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/wal
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/data/data
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:00.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:00.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.436880 31725 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:00.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.438596 31725 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:00.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.439013 31725 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:00.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.441905 31725 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:07:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.446300 31725 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:07:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.446512 31725 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:07:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.446714 31725 ts_tablet_manager.cc:616] Registered 0 tablets
21:07:00.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.446854 31725 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:07:00.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.603437 31725 rpc_server.cc:307] RPC server started. Bound to: 127.30.140.130:34739
21:07:00.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.603538 31858 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.140.130:34739 every 8 connection(s)
21:07:00.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.606647 31725 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/data/info.pb
21:07:00.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.613001 31282 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 31725
21:07:00.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.613629 31282 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-1/wal/instance
21:07:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.630628 31282 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/wal
21:07:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/data
21:07:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:00.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/logs
21:07:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/data/info.pb
21:07:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:00.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.131:0
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.140.131
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.131
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:00.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:07:00.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.667369 31859 heartbeater.cc:344] Connected to a master server at 127.30.140.188:44343
21:07:00.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.667992 31859 heartbeater.cc:461] Registering TS with master...
21:07:00.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.669492 31859 heartbeater.cc:507] Master 127.30.140.188:44343 requested a full tablet report, sending...
21:07:00.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.672871 31482 ts_manager.cc:194] Registered new tserver with Master: daf27ab253ce429ea73b8dc04b3f5b44 (127.30.140.130:34739)
21:07:00.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.675048 31482 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.140.130:33581
21:07:00.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.675621 31861 heartbeater.cc:344] Connected to a master server at 127.30.140.190:40487
21:07:00.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.675961 31861 heartbeater.cc:461] Registering TS with master...
21:07:00.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.677028 31861 heartbeater.cc:507] Master 127.30.140.190:40487 requested a full tablet report, sending...
21:07:00.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.679994 31326 ts_manager.cc:194] Registered new tserver with Master: daf27ab253ce429ea73b8dc04b3f5b44 (127.30.140.130:34739)
21:07:00.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.687104 31860 heartbeater.cc:344] Connected to a master server at 127.30.140.189:40703
21:07:00.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.687528 31860 heartbeater.cc:461] Registering TS with master...
21:07:00.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.688251 31860 heartbeater.cc:507] Master 127.30.140.189:40703 requested a full tablet report, sending...
21:07:00.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:00.691198 31402 ts_manager.cc:194] Registered new tserver with Master: daf27ab253ce429ea73b8dc04b3f5b44 (127.30.140.130:34739)
21:07:01.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:01.022071 31867 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:01.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:01.022653 31867 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:01.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:01.023186 31867 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:01.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:01.060395 31867 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:01.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:01.061689 31867 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.140.131
21:07:01.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:01.100466 31867 tablet_server_runner.cc:78] Tablet server non-default flags:
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.140.148:39037
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/data
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/wal
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.140.131:0
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/data/info.pb
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.140.131
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.140.190:40487,127.30.140.189:40703,127.30.140.188:44343
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.140.131
21:07:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/logs
21:07:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:07:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:07:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:07:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:07:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:01.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:01.102452 31867 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:01.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:01.104521 31867 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:01.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:01.118830 31875 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:01.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:01.679226 31859 heartbeater.cc:499] Master 127.30.140.188:44343 was elected leader, sending a full tablet report...
21:07:02.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:02.523969 31874 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 31867
21:07:02.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:02.535331 31867 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.416s user 0.496s sys 0.906s
21:07:02.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:02.535874 31867 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.417s user 0.496s sys 0.906s
21:07:02.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:01.119902 31876 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:02.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:02.537410 31877 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1416 milliseconds
21:07:02.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.537819 31867 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:02.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:02.537892 31878 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:02.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.541790 31867 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:02.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.544129 31867 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:02.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.545509 31867 hybrid_clock.cc:648] HybridClock initialized: now 1765573622545447 us; error 60 us; skew 500 ppm
21:07:02.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.546398 31867 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:02.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.553196 31867 webserver.cc:492] Webserver started at http://127.30.140.131:36941/ using document root <none> and password file <none>
21:07:02.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.554390 31867 fs_manager.cc:362] Metadata directory not provided
21:07:02.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.554644 31867 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:02.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.555217 31867 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:07:02.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.559804 31867 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/data/instance:
21:07:02.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3433b3d43424cf9a1579c147aa76002"
21:07:02.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:02 on dist-test-slave-vhkf"
21:07:02.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.561041 31867 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/wal/instance:
21:07:02.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3433b3d43424cf9a1579c147aa76002"
21:07:02.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:02 on dist-test-slave-vhkf"
21:07:02.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.568786 31867 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
21:07:02.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.574990 31886 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:02.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.576216 31867 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
21:07:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.576618 31867 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/wal
21:07:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c3433b3d43424cf9a1579c147aa76002"
21:07:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:02 on dist-test-slave-vhkf"
21:07:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.577013 31867 fs_report.cc:389] FS layout report
21:07:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/wal
21:07:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/wal
21:07:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/data/data
21:07:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:02.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:02.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:02.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:02.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:02.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:02.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:02.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:02.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:02.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:02.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.629624 31867 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:02.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.631331 31867 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:02.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.631904 31867 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:02.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.634744 31867 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:07:02.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.639662 31867 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:07:02.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.639932 31867 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:07:02.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.640251 31867 ts_tablet_manager.cc:616] Registered 0 tablets
21:07:02.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.640426 31867 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.813020 31867 rpc_server.cc:307] RPC server started. Bound to: 127.30.140.131:34683
21:07:02.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.813097 31998 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.140.131:34683 every 8 connection(s)
21:07:02.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.816916 31867 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/data/info.pb
21:07:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.826622 31282 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 31867
21:07:02.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.827229 31282 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster2823433410088551178/ts-2/wal/instance
21:07:02.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.861832 32000 heartbeater.cc:344] Connected to a master server at 127.30.140.189:40703
21:07:02.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.862416 32000 heartbeater.cc:461] Registering TS with master...
21:07:02.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.864002 32000 heartbeater.cc:507] Master 127.30.140.189:40703 requested a full tablet report, sending...
21:07:02.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.864197 31999 heartbeater.cc:344] Connected to a master server at 127.30.140.188:44343
21:07:02.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.864552 31999 heartbeater.cc:461] Registering TS with master...
21:07:02.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.865640 31999 heartbeater.cc:507] Master 127.30.140.188:44343 requested a full tablet report, sending...
21:07:02.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.867583 31402 ts_manager.cc:194] Registered new tserver with Master: c3433b3d43424cf9a1579c147aa76002 (127.30.140.131:34683)
21:07:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.868263 31482 ts_manager.cc:194] Registered new tserver with Master: c3433b3d43424cf9a1579c147aa76002 (127.30.140.131:34683)
21:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.868851 32001 heartbeater.cc:344] Connected to a master server at 127.30.140.190:40487
21:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.869292 32001 heartbeater.cc:461] Registering TS with master...
21:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.870368 32001 heartbeater.cc:507] Master 127.30.140.190:40487 requested a full tablet report, sending...
21:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.870362 31482 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.30.140.131:42307
21:07:02.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.873368 31326 ts_manager.cc:194] Registered new tserver with Master: c3433b3d43424cf9a1579c147aa76002 (127.30.140.131:34683)
21:07:02.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:02.886184 31282 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:07:02.898 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:07:02.901 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:07:02.913 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.30.140.190"
port: 40487
}
bound_http_address {
host: "127.30.140.190"
port: 43141
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.30.140.189"
port: 40703
}
bound_http_address {
host: "127.30.140.189"
port: 36573
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.30.140.188"
port: 44343
}
bound_http_address {
host: "127.30.140.188"
port: 40645
}
}
}
21:07:02.917 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:07:02.920 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.30.140.129"
port: 44733
}
bound_http_address {
host: "127.30.140.129"
port: 39035
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.30.140.130"
port: 34739
}
bound_http_address {
host: "127.30.140.130"
port: 42455
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.30.140.131"
port: 34683
}
bound_http_address {
host: "127.30.140.131"
port: 36941
}
}
}
21:07:02.920 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:07:03.100 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
21:07:03.127 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:07:03.261 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskGYhehh/test-tmp
21:07:03.262 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:07:03.262 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskGYhehh/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:07:03.266 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
21:07:03.641 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:03.641 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:07:03.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:03.644529 32010 mini_oidc.cc:150] Starting JWKS server
21:07:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:03.648736 32010 webserver.cc:492] Webserver started at http://127.0.0.1:42783/ using document root <none> and password file <none>
21:07:03.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:03.649397 32010 mini_oidc.cc:186] Starting OIDC Discovery server
21:07:03.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:03.651760 32010 webserver.cc:492] Webserver started at http://127.0.0.1:43373/ using document root <none> and password file <none>
21:07:03.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
21:07:03.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/principal' for realm 'KRBTEST.COM',
21:07:03.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
21:07:03.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:03 dist-test-slave-vhkf krb5kdc[32022](info): setting up network...
21:07:03.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
21:07:03.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:03 dist-test-slave-vhkf krb5kdc[32022](info): set up 2 sockets
21:07:03.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:03 dist-test-slave-vhkf krb5kdc[32022](info): commencing operation
21:07:03.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
21:07:03.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:03.874827 31999 heartbeater.cc:499] Master 127.30.140.188:44343 was elected leader, sending a full tablet report...
21:07:05.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:05.834964 32010 mini_kdc.cc:121] Time spent starting KDC: real 2.132s user 0.004s sys 0.017s
21:07:05.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
21:07:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:07:05.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
21:07:05.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
21:07:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:07:05.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
21:07:05.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
21:07:05.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:07:05.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
21:07:05.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:07:05.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/test-user.keytab.
21:07:05.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/test-user.keytab.
21:07:05.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:05 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573625, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:05.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:07:05.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-12T21:07:05Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:07:05.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-12T21:07:05Z Disabled control of system clock
21:07:06.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.31.66.190@KRBTEST.COM; defaulting to no policy
21:07:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:06.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.31.66.190@KRBTEST.COM" created.
21:07:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:06.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:06.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.31.66.190@KRBTEST.COM; defaulting to no policy
21:07:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:06.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.31.66.190@KRBTEST.COM" created.
21:07:06.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:06.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.190 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:06.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.190 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:06.086653 32010 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal
21:07:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data
21:07:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:06.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/logs
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data/info.pb
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.190:39107
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.190
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.190
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:06.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:07:06.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:06.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.424386 32039 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:06.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.424995 32039 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:06.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.425716 32039 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.460361 32039 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:07:06.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.460628 32039 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:07:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.460901 32039 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.461259 32039 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.461606 32039 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:07:06.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.461834 32039 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:06.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.461987 32039 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42783/jwks.json
21:07:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:06.501163 32039 master_runner.cc:386] Master server non-default flags:
21:07:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data
21:07:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal
21:07:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:07:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.190
21:07:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.190:39107
21:07:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data/info.pb
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.190
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/logs
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:06.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:06.502625 32039 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:06.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:06.503726 32039 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:06.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.517439 32045 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:07.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:06.517613 32046 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:07.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:07.858880 32047 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
21:07:07.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:07.861012 32048 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:07.864177 32039 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.347s user 0.538s sys 0.806s
21:07:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:07.864480 32039 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.348s user 0.538s sys 0.806s
21:07:07.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.864691 32039 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:07.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.865927 32039 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:07.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.868903 32039 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:07.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.872507 32039 hybrid_clock.cc:648] HybridClock initialized: now 1765573627872447 us; error 147 us; skew 500 ppm
21:07:07.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.873683 32039 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:07.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:07 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573627, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.883404 32039 init.cc:377] Logged in from keytab as kudu/127.31.66.190@KRBTEST.COM (short username kudu)
21:07:07.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.890378 32039 webserver.cc:492] Webserver started at http://127.31.66.190:35807/ using document root <none> and password file <none>
21:07:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.891762 32039 fs_manager.cc:362] Metadata directory not provided
21:07:07.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.892061 32039 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:07.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.892879 32039 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:07:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.899811 32039 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data/instance:
21:07:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "147b937cb1ed40a28e53cd67b23bf42f"
21:07:07.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:07 on dist-test-slave-vhkf"
21:07:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.901794 32039 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal/instance:
21:07:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "147b937cb1ed40a28e53cd67b23bf42f"
21:07:07.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:07 on dist-test-slave-vhkf"
21:07:07.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.916932 32039 fs_manager.cc:696] Time spent creating directory manager: real 0.014s user 0.008s sys 0.004s
21:07:07.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.926541 32056 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:07.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.928570 32039 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.001s sys 0.004s
21:07:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.929095 32039 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal
21:07:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "147b937cb1ed40a28e53cd67b23bf42f"
21:07:07.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:07 on dist-test-slave-vhkf"
21:07:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:07.929767 32039 fs_report.cc:389] FS layout report
21:07:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal
21:07:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal
21:07:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data/data
21:07:07.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:07.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:08.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.014645 32039 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:08.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.021147 32039 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:08.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.021802 32039 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:08.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.105813 32039 rpc_server.cc:307] RPC server started. Bound to: 127.31.66.190:39107
21:07:08.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.105943 32108 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.66.190:39107 every 8 connection(s)
21:07:08.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.108794 32039 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data/info.pb
21:07:08.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.115291 32109 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:07:08.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.119539 32010 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 32039
21:07:08.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.120095 32010 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal/instance
21:07:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.127493 32109 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } has no permanent_uuid. Determining permanent_uuid...
21:07:08.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.31.66.189@KRBTEST.COM; defaulting to no policy
21:07:08.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:08.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.31.66.189@KRBTEST.COM" created.
21:07:08.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:08 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573627, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.190@KRBTEST.COM for kudu/127.31.66.190@KRBTEST.COM
21:07:08.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:08.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:08.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:08.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.172065 32109 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } has no permanent_uuid. Determining permanent_uuid...
21:07:08.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.175280 32058 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.66.189:33973: connect: Connection refused (error 111)
21:07:08.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.181711 32109 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.189:33973: Network error: Client connection negotiation failed: client connection to 127.31.66.189:33973: connect: Connection refused (error 111)
21:07:08.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.31.66.189@KRBTEST.COM; defaulting to no policy
21:07:08.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:08.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.31.66.189@KRBTEST.COM" created.
21:07:08.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:08.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.189 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:08.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.189 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.218688 32010 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/logs
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data/info.pb
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.189:33973
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.189
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.189
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:07:08.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.231316 32109 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } attempt: 1
21:07:08.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.235734 32109 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.189:33973: Network error: Client connection negotiation failed: client connection to 127.31.66.189:33973: connect: Connection refused (error 111)
21:07:08.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.304325 32109 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } attempt: 2
21:07:08.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.308897 32109 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.189:33973: Network error: Client connection negotiation failed: client connection to 127.31.66.189:33973: connect: Connection refused (error 111)
21:07:08.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.400514 32109 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } attempt: 3
21:07:08.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.407514 32109 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.189:33973: Network error: Client connection negotiation failed: client connection to 127.31.66.189:33973: connect: Connection refused (error 111)
21:07:08.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.550987 32109 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } attempt: 4
21:07:08.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.555683 32109 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.189:33973: Network error: Client connection negotiation failed: client connection to 127.31.66.189:33973: connect: Connection refused (error 111)
21:07:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.568018 32118 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:08.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.568665 32118 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:08.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.569453 32118 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:08.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.610750 32118 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:07:08.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.611124 32118 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:07:08.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.611644 32118 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:08.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.612010 32118 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:08.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.612361 32118 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:07:08.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.612687 32118 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:08.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.612933 32118 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42783/jwks.json
21:07:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.656929 32118 master_runner.cc:386] Master server non-default flags:
21:07:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data
21:07:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal
21:07:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:07:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:08.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.189
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.189:33973
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data/info.pb
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.189
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/logs
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:08.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:07:08.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:08.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:07:08.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:07:08.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:08.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.658905 32118 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:08.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.660322 32118 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:08.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.672994 32126 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:08.855219 32109 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } attempt: 5
21:07:08.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.871222 32109 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.189:33973: Network error: Client connection negotiation failed: client connection to 127.31.66.189:33973: connect: Connection refused (error 111)
21:07:09.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:09.418897 32109 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } attempt: 6
21:07:09.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:09.452734 32109 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.189:33973: Network error: Client connection negotiation failed: client connection to 127.31.66.189:33973: connect: Connection refused (error 111)
21:07:10.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:10.077062 32125 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 32118
21:07:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:10.416023 32118 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.741s user 0.577s sys 1.158s
21:07:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:10.417029 32118 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.743s user 0.577s sys 1.158s
21:07:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:08.674914 32127 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:10.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:10.418291 32128 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1741 milliseconds
21:07:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.419112 32118 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:10.419198 32129 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:10.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.423264 32118 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:10.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.426230 32118 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:10.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.427657 32118 hybrid_clock.cc:648] HybridClock initialized: now 1765573630427611 us; error 55 us; skew 500 ppm
21:07:10.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.428642 32118 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:10.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:10 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573630, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:10.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.440130 32118 init.cc:377] Logged in from keytab as kudu/127.31.66.189@KRBTEST.COM (short username kudu)
21:07:10.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.445933 32118 webserver.cc:492] Webserver started at http://127.31.66.189:44435/ using document root <none> and password file <none>
21:07:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.447059 32118 fs_manager.cc:362] Metadata directory not provided
21:07:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.447269 32118 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.447711 32118 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:07:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.452747 32118 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data/instance:
21:07:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06fdd9cd41a0441b93eb0061d60b7672"
21:07:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:10 on dist-test-slave-vhkf"
21:07:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.454169 32118 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal/instance:
21:07:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06fdd9cd41a0441b93eb0061d60b7672"
21:07:10.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:10 on dist-test-slave-vhkf"
21:07:10.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.463300 32118 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.004s sys 0.004s
21:07:10.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.470443 32140 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:10.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.471720 32118 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.004s
21:07:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.472069 32118 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal
21:07:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06fdd9cd41a0441b93eb0061d60b7672"
21:07:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:10 on dist-test-slave-vhkf"
21:07:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.472405 32118 fs_report.cc:389] FS layout report
21:07:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal
21:07:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal
21:07:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data/data
21:07:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:10.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:10.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:10.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.513466 32109 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } attempt: 7
21:07:10.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:10.522091 32109 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.189:33973: Network error: Client connection negotiation failed: client connection to 127.31.66.189:33973: connect: Connection refused (error 111)
21:07:10.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.551023 32118 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.560771 32118 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:10.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.561270 32118 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:10.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.649333 32118 rpc_server.cc:307] RPC server started. Bound to: 127.31.66.189:33973
21:07:10.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.649492 32193 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.66.189:33973 every 8 connection(s)
21:07:10.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.652393 32118 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data/info.pb
21:07:10.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.654621 32010 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 32118
21:07:10.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.655125 32010 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal/instance
21:07:10.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.664451 32194 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:07:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.31.66.188@KRBTEST.COM; defaulting to no policy
21:07:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:10.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.31.66.188@KRBTEST.COM" created.
21:07:10.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.683012 32194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } has no permanent_uuid. Determining permanent_uuid...
21:07:10.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:10.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:10.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:10.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:10 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573630, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.189@KRBTEST.COM for kudu/127.31.66.190@KRBTEST.COM
21:07:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.31.66.188@KRBTEST.COM; defaulting to no policy
21:07:10.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.31.66.188@KRBTEST.COM" created.
21:07:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.723670 32194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } has no permanent_uuid. Determining permanent_uuid...
21:07:10.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:10 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573630, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.189@KRBTEST.COM for kudu/127.31.66.189@KRBTEST.COM
21:07:10.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:10.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.188 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:10.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.188 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.751911 32010 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal
21:07:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data
21:07:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/logs
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data/info.pb
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.188:39381
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.188
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.188
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:07:10.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:10.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.756927 32194 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } has no permanent_uuid. Determining permanent_uuid...
21:07:10.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:10.760032 32145 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:10.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:10.765386 32194 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.188:39381: Network error: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.814989 32194 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } attempt: 1
21:07:10.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:10.819813 32194 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.188:39381: Network error: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:10.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.888252 32194 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } attempt: 2
21:07:10.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:10.898665 32194 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.188:39381: Network error: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:10.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:10.990175 32194 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } attempt: 3
21:07:10.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:10.995014 32194 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.188:39381: Network error: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.133765 32203 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:11.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.134512 32203 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:11.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.135311 32203 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:11.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:11.138429 32194 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } attempt: 4
21:07:11.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.142802 32194 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.188:39381: Network error: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.173977 32203 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:07:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.174288 32203 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:07:11.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.174734 32203 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:11.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.175063 32203 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:11.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.175379 32203 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:07:11.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.175721 32203 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:11.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.175957 32203 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42783/jwks.json
21:07:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:11.217345 32203 master_runner.cc:386] Master server non-default flags:
21:07:11.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data
21:07:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal
21:07:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:07:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:11.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.188
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.188:39381
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data/info.pb
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.188
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/logs
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:11.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:07:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:07:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:11.219184 32203 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:11.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:11.220640 32203 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:11.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.233017 32211 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:11.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:11.442318 32194 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } attempt: 5
21:07:11.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.450906 32194 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.188:39381: Network error: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:11.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:11.998564 32194 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } attempt: 6
21:07:12.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:12.010988 32194 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.188:39381: Network error: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.234603 32212 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:12.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:11.243434 32214 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:12.561636 32213 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1323 milliseconds
21:07:12.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.561777 32203 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:12.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.563162 32203 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:12.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.565979 32203 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:12.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.567368 32203 hybrid_clock.cc:648] HybridClock initialized: now 1765573632567331 us; error 69 us; skew 500 ppm
21:07:12.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.568310 32203 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:12.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:12 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573632, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:12.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.575273 32203 init.cc:377] Logged in from keytab as kudu/127.31.66.188@KRBTEST.COM (short username kudu)
21:07:12.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.581455 32203 webserver.cc:492] Webserver started at http://127.31.66.188:46593/ using document root <none> and password file <none>
21:07:12.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.582602 32203 fs_manager.cc:362] Metadata directory not provided
21:07:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.582854 32203 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:12.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.583423 32203 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:07:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.588053 32203 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data/instance:
21:07:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99f382df85844e529eb3e6a120dcb1ee"
21:07:12.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:12 on dist-test-slave-vhkf"
21:07:12.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.589474 32203 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal/instance:
21:07:12.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99f382df85844e529eb3e6a120dcb1ee"
21:07:12.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:12 on dist-test-slave-vhkf"
21:07:12.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.597666 32203 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
21:07:12.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.603958 32225 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:12.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.605350 32203 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
21:07:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.605758 32203 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal
21:07:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99f382df85844e529eb3e6a120dcb1ee"
21:07:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:12 on dist-test-slave-vhkf"
21:07:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.606211 32203 fs_report.cc:389] FS layout report
21:07:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal
21:07:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal
21:07:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data/data
21:07:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:12.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:12.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:12.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:12.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:12.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:12.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:12.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:12.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:12.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:12.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:12.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.612615 32109 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } attempt: 8
21:07:12.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:12 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573627, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.190@KRBTEST.COM for kudu/127.31.66.189@KRBTEST.COM
21:07:12.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.645725 32109 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } has no permanent_uuid. Determining permanent_uuid...
21:07:12.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:12.667302 32109 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.188:39381: Network error: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:12.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.710578 32203 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:12.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.717327 32203 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:12.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.718081 32203 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:12.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.732867 32109 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } attempt: 1
21:07:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:12.737826 32109 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.188:39381: Network error: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:12.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.791280 32109 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } attempt: 2
21:07:12.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:12.798228 32109 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.31.66.188:39381: Network error: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:12.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.813787 32203 rpc_server.cc:307] RPC server started. Bound to: 127.31.66.188:39381
21:07:12.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.813975 32280 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.66.188:39381 every 8 connection(s)
21:07:12.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.817782 32203 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data/info.pb
21:07:12.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.826227 32281 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:07:12.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.828318 32010 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 32203
21:07:12.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.828840 32010 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal/instance
21:07:12.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.843641 32281 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } has no permanent_uuid. Determining permanent_uuid...
21:07:12.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:12 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573625, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.31.66.190@KRBTEST.COM
21:07:12.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:12 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573632, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.188@KRBTEST.COM for kudu/127.31.66.190@KRBTEST.COM
21:07:12.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.874775 32109 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } attempt: 3
21:07:12.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.886596 32281 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } has no permanent_uuid. Determining permanent_uuid...
21:07:12.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:12 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573625, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.31.66.189@KRBTEST.COM
21:07:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:12 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573627, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.190@KRBTEST.COM for kudu/127.31.66.188@KRBTEST.COM
21:07:12.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:12 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573632, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.188@KRBTEST.COM for kudu/127.31.66.189@KRBTEST.COM
21:07:12.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:12 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573625, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.31.66.188@KRBTEST.COM
21:07:12.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.926235 32109 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: Bootstrap starting.
21:07:12.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.926227 32281 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } has no permanent_uuid. Determining permanent_uuid...
21:07:12.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.935688 32109 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: Neither blocks nor log segments found. Creating new log.
21:07:12.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.938428 32109 log.cc:826] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: Log is configured to *not* fsync() on all Append() calls
21:07:12.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:12 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573632, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.188@KRBTEST.COM for kudu/127.31.66.188@KRBTEST.COM
21:07:12.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.945297 32109 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: No bootstrap required, opened a new log
21:07:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.969185 32281 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: Bootstrap starting.
21:07:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.974073 32109 raft_consensus.cc:359] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.975613 32109 raft_consensus.cc:385] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:07:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.976120 32109 raft_consensus.cc:740] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 147b937cb1ed40a28e53cd67b23bf42f, State: Initialized, Role: FOLLOWER
21:07:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.977428 32109 consensus_queue.cc:260] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.978183 32281 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: Neither blocks nor log segments found. Creating new log.
21:07:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.981318 32281 log.cc:826] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: Log is configured to *not* fsync() on all Append() calls
21:07:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.981618 32291 sys_catalog.cc:455] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:12.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.982481 32291 sys_catalog.cc:458] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: This master's current role is: FOLLOWER
21:07:12.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.983783 32109 sys_catalog.cc:565] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: configured and running, proceeding with master startup.
21:07:12.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:12.987730 32281 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: No bootstrap required, opened a new log
21:07:13.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.020507 32281 raft_consensus.cc:359] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.021708 32281 raft_consensus.cc:385] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:07:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.022050 32281 raft_consensus.cc:740] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 99f382df85844e529eb3e6a120dcb1ee, State: Initialized, Role: FOLLOWER
21:07:13.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:13.022449 32303 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:07:13.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:13.022818 32303 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:07:13.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.023373 32281 consensus_queue.cc:260] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:13.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.031713 32305 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:13.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.036880 32305 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: This master's current role is: FOLLOWER
21:07:13.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.035717 32281 sys_catalog.cc:565] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: configured and running, proceeding with master startup.
21:07:13.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:13.040721 32063 tablet.cc:2378] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:07:13.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.018708 32039 master_runner.cc:186] Error getting master registration for 127.31.66.189:33973: OK, instance_id { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" instance_seqno: 1765573630626576 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:07:13.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.057585 32039 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:07:13.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.071661 32194 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } attempt: 7
21:07:13.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:13 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573630, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.189@KRBTEST.COM for kudu/127.31.66.188@KRBTEST.COM
21:07:13.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:13.128180 32317 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:07:13.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:13.128567 32317 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:07:13.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.145404 32194 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: Bootstrap starting.
21:07:13.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.154781 32194 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: Neither blocks nor log segments found. Creating new log.
21:07:13.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.157819 32194 log.cc:826] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: Log is configured to *not* fsync() on all Append() calls
21:07:13.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.115137 32203 master_runner.cc:186] Error getting master registration for 127.31.66.189:33973: OK, instance_id { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" instance_seqno: 1765573630626576 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:07:13.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.186322 32203 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:07:13.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.186829 32194 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: No bootstrap required, opened a new log
21:07:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.210521 32194 raft_consensus.cc:359] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:13.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.211352 32194 raft_consensus.cc:385] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:07:13.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.211666 32194 raft_consensus.cc:740] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 06fdd9cd41a0441b93eb0061d60b7672, State: Initialized, Role: FOLLOWER
21:07:13.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.212529 32194 consensus_queue.cc:260] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:13.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.216408 32320 sys_catalog.cc:455] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:13.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.217453 32320 sys_catalog.cc:458] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: This master's current role is: FOLLOWER
21:07:13.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.218241 32194 sys_catalog.cc:565] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: configured and running, proceeding with master startup.
21:07:13.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:13.229552 32235 tablet.cc:2378] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:07:13.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.264569 32291 raft_consensus.cc:493] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:07:13.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.265381 32291 raft_consensus.cc:515] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:13.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.270993 32169 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "147b937cb1ed40a28e53cd67b23bf42f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "06fdd9cd41a0441b93eb0061d60b7672" is_pre_election: true
21:07:13.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.272217 32169 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 147b937cb1ed40a28e53cd67b23bf42f in term 0.
21:07:13.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.279017 32058 leader_election.cc:304] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 06fdd9cd41a0441b93eb0061d60b7672, 147b937cb1ed40a28e53cd67b23bf42f; no voters:
21:07:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.288681 32256 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "147b937cb1ed40a28e53cd67b23bf42f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "99f382df85844e529eb3e6a120dcb1ee" is_pre_election: true
21:07:13.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.290086 32256 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 147b937cb1ed40a28e53cd67b23bf42f in term 0.
21:07:13.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.294041 32291 leader_election.cc:290] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 06fdd9cd41a0441b93eb0061d60b7672 (127.31.66.189:33973), 99f382df85844e529eb3e6a120dcb1ee (127.31.66.188:39381)
21:07:13.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.297871 32291 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 0 FOLLOWER]: Leader pre-election won for term 1
21:07:13.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.301115 32291 raft_consensus.cc:493] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:07:13.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.301680 32291 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 0 FOLLOWER]: Advancing to term 1
21:07:13.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:13.320580 32148 tablet.cc:2378] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:07:13.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:13.330066 32058 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.31.66.189:33973, user_credentials={real_user=slave}} blocked reactor thread for 51203.1us
21:07:13.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.336007 32291 raft_consensus.cc:515] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.349712 32169 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "147b937cb1ed40a28e53cd67b23bf42f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "06fdd9cd41a0441b93eb0061d60b7672"
21:07:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.350524 32169 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 0 FOLLOWER]: Advancing to term 1
21:07:13.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.363368 32169 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 147b937cb1ed40a28e53cd67b23bf42f in term 1.
21:07:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.366631 32256 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "147b937cb1ed40a28e53cd67b23bf42f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "99f382df85844e529eb3e6a120dcb1ee"
21:07:13.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.367461 32256 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 0 FOLLOWER]: Advancing to term 1
21:07:13.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.378374 32256 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 147b937cb1ed40a28e53cd67b23bf42f in term 1.
21:07:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.379326 32291 leader_election.cc:290] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [CANDIDATE]: Term 1 election: Requested vote from peers 06fdd9cd41a0441b93eb0061d60b7672 (127.31.66.189:33973), 99f382df85844e529eb3e6a120dcb1ee (127.31.66.188:39381)
21:07:13.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.383416 32058 leader_election.cc:304] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 06fdd9cd41a0441b93eb0061d60b7672, 147b937cb1ed40a28e53cd67b23bf42f; no voters:
21:07:13.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.392704 32291 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 1 FOLLOWER]: Leader election won for term 1
21:07:13.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.396555 32291 raft_consensus.cc:697] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 1 LEADER]: Becoming Leader. State: Replica: 147b937cb1ed40a28e53cd67b23bf42f, State: Running, Role: LEADER
21:07:13.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.398227 32291 consensus_queue.cc:237] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.420226 32327 sys_catalog.cc:455] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 147b937cb1ed40a28e53cd67b23bf42f. Latest consensus state: current_term: 1 leader_uuid: "147b937cb1ed40a28e53cd67b23bf42f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.421031 32327 sys_catalog.cc:458] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: This master's current role is: LEADER
21:07:13.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:13.422523 32332 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:07:13.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:13.422907 32332 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:07:13.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.431284 32335 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:07:13.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.442139 32335 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:07:13.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.478479 32256 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Refusing update from remote peer 147b937cb1ed40a28e53cd67b23bf42f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:07:13.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.480444 32327 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [LEADER]: Connected to new peer: Peer: permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:07:13.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.493418 32169 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 1 FOLLOWER]: Refusing update from remote peer 147b937cb1ed40a28e53cd67b23bf42f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:07:13.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.495569 32327 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [LEADER]: Connected to new peer: Peer: permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:07:13.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.523876 32305 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: SysCatalogTable state changed. Reason: New leader 147b937cb1ed40a28e53cd67b23bf42f. Latest consensus state: current_term: 1 leader_uuid: "147b937cb1ed40a28e53cd67b23bf42f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:13.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.524673 32305 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: This master's current role is: FOLLOWER
21:07:13.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.535312 32336 mvcc.cc:204] Tried to move back new op lower bound from 7231789602704470016 to 7231789602422947840. Current Snapshot: MvccSnapshot[applied={T|T < 7231789602704470016}]
21:07:13.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.539953 32320 sys_catalog.cc:455] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 147b937cb1ed40a28e53cd67b23bf42f. Latest consensus state: current_term: 1 leader_uuid: "147b937cb1ed40a28e53cd67b23bf42f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:13.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.540776 32320 sys_catalog.cc:458] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: This master's current role is: FOLLOWER
21:07:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.546614 32305 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "147b937cb1ed40a28e53cd67b23bf42f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:13.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.547509 32305 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: This master's current role is: FOLLOWER
21:07:13.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.562819 32338 mvcc.cc:204] Tried to move back new op lower bound from 7231789602704470016 to 7231789602422947840. Current Snapshot: MvccSnapshot[applied={T|T < 7231789602704470016}]
21:07:13.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.563836 32327 sys_catalog.cc:455] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "147b937cb1ed40a28e53cd67b23bf42f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:13.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.564611 32327 sys_catalog.cc:458] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: This master's current role is: LEADER
21:07:13.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.565685 32327 sys_catalog.cc:455] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "147b937cb1ed40a28e53cd67b23bf42f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:13.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.566587 32327 sys_catalog.cc:458] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: This master's current role is: LEADER
21:07:13.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.567487 32320 sys_catalog.cc:455] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "147b937cb1ed40a28e53cd67b23bf42f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:13.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.569595 32320 sys_catalog.cc:458] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: This master's current role is: FOLLOWER
21:07:13.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.577713 32335 catalog_manager.cc:1357] Generated new cluster ID: 50892035360048b5a5203f4bbf222c78
21:07:13.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.578224 32335 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:07:13.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.690692 32335 catalog_manager.cc:1380] Generated new certificate authority record
21:07:13.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.693070 32335 catalog_manager.cc:1514] Loading token signing keys...
21:07:13.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.799772 32335 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: Generated new TSK 0
21:07:13.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:13.801376 32335 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:07:13.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.31.66.129@KRBTEST.COM; defaulting to no policy
21:07:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:13.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.31.66.129@KRBTEST.COM" created.
21:07:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.129 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:14.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.129 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:14.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:14.002602 32010 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.31.66.129: real 0.134s user 0.008s sys 0.007s
21:07:14.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.31.66.129@KRBTEST.COM; defaulting to no policy
21:07:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:14.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.31.66.129@KRBTEST.COM" created.
21:07:14.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:14.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.129 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:14.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.129 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:14.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:14.104525 32010 mini_kdc.cc:268] Time spent creating service keytab for HTTP/127.31.66.129: real 0.101s user 0.003s sys 0.010s
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:14.106968 32010 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/wal
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/data
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/logs
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/data/info.pb
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.129:0
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.66.129
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.129
21:07:14.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.129
21:07:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:14.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:14.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:14.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:14.135411 32317 catalog_manager.cc:1269] Loaded cluster ID: 50892035360048b5a5203f4bbf222c78
21:07:14.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:14.135744 32317 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: loading cluster ID for follower catalog manager: success
21:07:14.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:14.150101 32317 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: acquiring CA information for follower catalog manager: success
21:07:14.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:14.159731 32317 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:07:14.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:14.429334 32332 catalog_manager.cc:1269] Loaded cluster ID: 50892035360048b5a5203f4bbf222c78
21:07:14.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:14.430003 32332 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: loading cluster ID for follower catalog manager: success
21:07:14.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:14.446116 32332 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: acquiring CA information for follower catalog manager: success
21:07:14.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:14.458556 32332 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:07:14.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:14.723852 32351 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:14.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:14.724598 32351 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:14.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:14.725772 32351 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:14.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:14.865654 32351 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:14.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:14.867430 32351 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.66.129
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:15.008828 32351 tablet_server_runner.cc:78] Tablet server non-default flags:
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/data
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/wal
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.129
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.129:0
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:15.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/data/info.pb
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.129
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.66.129
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/logs
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:07:15.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:15.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:15.010937 32351 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:15.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:15.012526 32351 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:15.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:15.044539 32359 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:17.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:16.437563 32358 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 32351
21:07:17.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:17.533496 32351 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.485s user 0.595s sys 1.044s
21:07:17.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:15.051057 32360 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:17.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:17.533859 32351 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.486s user 0.595s sys 1.044s
21:07:17.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:17.545189 32367 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:17.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.545228 32351 server_base.cc:1047] running on GCE node
21:07:17.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.547801 32351 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:17.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.550606 32351 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:17.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.552059 32351 hybrid_clock.cc:648] HybridClock initialized: now 1765573637552010 us; error 73 us; skew 500 ppm
21:07:17.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.552881 32351 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:17.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:17 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573637, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:17.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.560068 32351 init.cc:377] Logged in from keytab as kudu/127.31.66.129@KRBTEST.COM (short username kudu)
21:07:17.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.566184 32351 webserver.cc:492] Webserver started at http://127.31.66.129:32897/ using document root <none> and password file <none>
21:07:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.567225 32351 fs_manager.cc:362] Metadata directory not provided
21:07:17.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.567472 32351 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:17.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.568020 32351 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:07:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.572530 32351 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/data/instance:
21:07:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0de102ab31a24754bfc4cec032df3368"
21:07:17.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:17 on dist-test-slave-vhkf"
21:07:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.573819 32351 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/wal/instance:
21:07:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0de102ab31a24754bfc4cec032df3368"
21:07:17.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:17 on dist-test-slave-vhkf"
21:07:17.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.581785 32351 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
21:07:17.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.587714 32374 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.589078 32351 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.003s
21:07:17.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.589471 32351 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/wal
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "0de102ab31a24754bfc4cec032df3368"
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:17 on dist-test-slave-vhkf"
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.589912 32351 fs_report.cc:389] FS layout report
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/wal
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/wal
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/data/data
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:17.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:17.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:17.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:17.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:17.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:17.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:17.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:17.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.659476 32351 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:17.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.673128 32351 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:17.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.673802 32351 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:17.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.688602 32351 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:07:17.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.694900 32351 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:07:17.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.695225 32351 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:07:17.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.695536 32351 ts_tablet_manager.cc:616] Registered 0 tablets
21:07:17.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.695746 32351 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:07:17.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:17 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573637, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:17.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.889721 32052 init.cc:295] Successfully reacquired a new kerberos TGT
21:07:17.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.961513 32351 rpc_server.cc:307] RPC server started. Bound to: 127.31.66.129:42691
21:07:17.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.961668 32487 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.66.129:42691 every 8 connection(s)
21:07:17.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.964517 32351 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/data/info.pb
21:07:17.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.970616 32010 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 32351
21:07:17.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:17.971256 32010 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-0/wal/instance
21:07:18.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:18 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573637, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.129@KRBTEST.COM for kudu/127.31.66.189@KRBTEST.COM
21:07:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.31.66.130@KRBTEST.COM; defaulting to no policy
21:07:18.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:18 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573637, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.129@KRBTEST.COM for kudu/127.31.66.188@KRBTEST.COM
21:07:18.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:18.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.31.66.130@KRBTEST.COM" created.
21:07:18.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:18 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573637, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.129@KRBTEST.COM for kudu/127.31.66.190@KRBTEST.COM
21:07:18.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.055104 32489 heartbeater.cc:344] Connected to a master server at 127.31.66.189:33973
21:07:18.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.055712 32489 heartbeater.cc:461] Registering TS with master...
21:07:18.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.057329 32489 heartbeater.cc:507] Master 127.31.66.189:33973 requested a full tablet report, sending...
21:07:18.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.061131 32159 ts_manager.cc:194] Registered new tserver with Master: 0de102ab31a24754bfc4cec032df3368 (127.31.66.129:42691)
21:07:18.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.063016 32488 heartbeater.cc:344] Connected to a master server at 127.31.66.188:39381
21:07:18.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.063412 32488 heartbeater.cc:461] Registering TS with master...
21:07:18.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.064514 32488 heartbeater.cc:507] Master 127.31.66.188:39381 requested a full tablet report, sending...
21:07:18.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.072290 32490 heartbeater.cc:344] Connected to a master server at 127.31.66.190:39107
21:07:18.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.072697 32490 heartbeater.cc:461] Registering TS with master...
21:07:18.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.073796 32490 heartbeater.cc:507] Master 127.31.66.190:39107 requested a full tablet report, sending...
21:07:18.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.073935 32246 ts_manager.cc:194] Registered new tserver with Master: 0de102ab31a24754bfc4cec032df3368 (127.31.66.129:42691)
21:07:18.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.077286 32073 ts_manager.cc:194] Registered new tserver with Master: 0de102ab31a24754bfc4cec032df3368 (127.31.66.129:42691)
21:07:18.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.081032 32073 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.31.66.129@KRBTEST.COM'} at 127.31.66.129:42549
21:07:18.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.130 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:18.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.130 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:18.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.31.66.130@KRBTEST.COM; defaulting to no policy
21:07:18.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:18.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.31.66.130@KRBTEST.COM" created.
21:07:18.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.130 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:18.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.130 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.137905 32010 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/wal
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/data
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/logs
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/data/info.pb
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.130:0
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.66.130
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.130
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:18.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.130
21:07:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:18.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:18.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:18.486223 32502 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:18.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:18.486757 32502 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:18.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:18.487505 32502 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:18.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:18.523227 32502 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:18.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:18.524278 32502 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.66.130
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.568468 32502 tablet_server_runner.cc:78] Tablet server non-default flags:
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/data
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/wal
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.130
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.130:0
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/data/info.pb
21:07:18.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.130
21:07:18.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:18.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:18.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:18.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.66.130
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/logs
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:18.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.570153 32502 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:18.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:18.571745 32502 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:18.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:18.586531 32511 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:19.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:19.086339 32490 heartbeater.cc:499] Master 127.31.66.190:39107 was elected leader, sending a full tablet report...
21:07:20.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:18.588483 32512 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:20.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:20.019426 32514 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:20.021579 32513 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1429 milliseconds
21:07:20.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.021742 32502 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:20.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.023116 32502 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:20.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.026413 32502 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:20.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.027804 32502 hybrid_clock.cc:648] HybridClock initialized: now 1765573640027758 us; error 80 us; skew 500 ppm
21:07:20.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.028920 32502 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:20.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:20 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573640, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:20.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.038188 32502 init.cc:377] Logged in from keytab as kudu/127.31.66.130@KRBTEST.COM (short username kudu)
21:07:20.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.047611 32502 webserver.cc:492] Webserver started at http://127.31.66.130:39171/ using document root <none> and password file <none>
21:07:20.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.048707 32502 fs_manager.cc:362] Metadata directory not provided
21:07:20.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.048947 32502 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:20.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.049569 32502 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:07:20.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.054720 32502 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/data/instance:
21:07:20.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6af2538206534e97afc6f93aa8201ce3"
21:07:20.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:20 on dist-test-slave-vhkf"
21:07:20.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.056022 32502 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/wal/instance:
21:07:20.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6af2538206534e97afc6f93aa8201ce3"
21:07:20.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:20 on dist-test-slave-vhkf"
21:07:20.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.067059 32502 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.011s sys 0.001s
21:07:20.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.073439 32522 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:20.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.075042 32502 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.003s sys 0.001s
21:07:20.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.075505 32502 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/wal
21:07:20.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6af2538206534e97afc6f93aa8201ce3"
21:07:20.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:20 on dist-test-slave-vhkf"
21:07:20.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.075941 32502 fs_report.cc:389] FS layout report
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/wal
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/wal
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/data/data
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:20.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.128994 32502 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.135084 32502 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:20.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.135579 32502 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:20.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.138298 32502 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:07:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.142979 32502 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:07:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.143198 32502 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:07:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.143401 32502 ts_tablet_manager.cc:616] Registered 0 tablets
21:07:20.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.143533 32502 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:07:20.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:20 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573640, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:20.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.448196 32136 init.cc:295] Successfully reacquired a new kerberos TGT
21:07:20.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.516496 32502 rpc_server.cc:307] RPC server started. Bound to: 127.31.66.130:42951
21:07:20.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.516848 32635 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.66.130:42951 every 8 connection(s)
21:07:20.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.519677 32502 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/data/info.pb
21:07:20.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.522369 32010 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 32502
21:07:20.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.522912 32010 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-1/wal/instance
21:07:20.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.31.66.131@KRBTEST.COM; defaulting to no policy
21:07:20.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:20.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.31.66.131@KRBTEST.COM" created.
21:07:20.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:20 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573640, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.130@KRBTEST.COM for kudu/127.31.66.188@KRBTEST.COM
21:07:20.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:20 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573640, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.130@KRBTEST.COM for kudu/127.31.66.189@KRBTEST.COM
21:07:20.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:20 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573640, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.130@KRBTEST.COM for kudu/127.31.66.190@KRBTEST.COM
21:07:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.616328 32638 heartbeater.cc:344] Connected to a master server at 127.31.66.190:39107
21:07:20.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.616999 32638 heartbeater.cc:461] Registering TS with master...
21:07:20.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.619045 32638 heartbeater.cc:507] Master 127.31.66.190:39107 requested a full tablet report, sending...
21:07:20.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.622473 32637 heartbeater.cc:344] Connected to a master server at 127.31.66.189:33973
21:07:20.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.623126 32637 heartbeater.cc:461] Registering TS with master...
21:07:20.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.623378 32073 ts_manager.cc:194] Registered new tserver with Master: 6af2538206534e97afc6f93aa8201ce3 (127.31.66.130:42951)
21:07:20.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.624624 32637 heartbeater.cc:507] Master 127.31.66.189:33973 requested a full tablet report, sending...
21:07:20.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.625855 32073 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.31.66.130@KRBTEST.COM'} at 127.31.66.130:58895
21:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.628335 32636 heartbeater.cc:344] Connected to a master server at 127.31.66.188:39381
21:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.628835 32636 heartbeater.cc:461] Registering TS with master...
21:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.629042 32159 ts_manager.cc:194] Registered new tserver with Master: 6af2538206534e97afc6f93aa8201ce3 (127.31.66.130:42951)
21:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.630599 32636 heartbeater.cc:507] Master 127.31.66.188:39381 requested a full tablet report, sending...
21:07:20.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.634078 32246 ts_manager.cc:194] Registered new tserver with Master: 6af2538206534e97afc6f93aa8201ce3 (127.31.66.130:42951)
21:07:20.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:20.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.131 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:20.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.31.66.131 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:20.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:20.637391 32010 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.31.66.131: real 0.103s user 0.000s sys 0.026s
21:07:20.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.31.66.131@KRBTEST.COM; defaulting to no policy
21:07:20.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:20.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.31.66.131@KRBTEST.COM" created.
21:07:20.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:20.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.131 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:20.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.31.66.131 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab.
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:20.685976 32010 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/wal
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/data
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/logs
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/data/info.pb
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.131:0
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.66.131
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.131
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.131
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:20.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:21.194664 32650 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:21.195173 32650 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:21.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:21.195766 32650 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:21.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:21.230396 32650 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:21.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:21.231407 32650 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.31.66.131
21:07:21.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:21.271309 32650 tablet_server_runner.cc:78] Tablet server non-default flags:
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/data
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/wal
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.131
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.131:0
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/data/info.pb
21:07:21.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.131
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.31.66.131
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/logs
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:21.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:21.272986 32650 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:21.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:21.274485 32650 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:21.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:21.289067 32658 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:21.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:21.633661 32638 heartbeater.cc:499] Master 127.31.66.190:39107 was elected leader, sending a full tablet report...
21:07:22.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:22 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573642, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:22.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:22.582852 32221 init.cc:295] Successfully reacquired a new kerberos TGT
21:07:22.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:22.673816 32632 debug-util.cc:398] Leaking SignalData structure 0x7b08000ae060 after lost signal to thread 32504
21:07:22.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:22.675604 32632 debug-util.cc:398] Leaking SignalData structure 0x7b08000aef60 after lost signal to thread 32635
21:07:23.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:21.291188 32659 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:23.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:23.156256 32661 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:23.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:23.165266 32660 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1862 milliseconds
21:07:23.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.165405 32650 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:23.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.167655 32650 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:23.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.171784 32650 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:23.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.175609 32650 hybrid_clock.cc:648] HybridClock initialized: now 1765573643175515 us; error 90 us; skew 500 ppm
21:07:23.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.176937 32650 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:23.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:23 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573643, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:23.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.187490 32650 init.cc:377] Logged in from keytab as kudu/127.31.66.131@KRBTEST.COM (short username kudu)
21:07:23.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.199069 32650 webserver.cc:492] Webserver started at http://127.31.66.131:41903/ using document root <none> and password file <none>
21:07:23.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.200644 32650 fs_manager.cc:362] Metadata directory not provided
21:07:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.200981 32650 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:23.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.201733 32650 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:07:23.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.210109 32650 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/data/instance:
21:07:23.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd1e6f298ab04727a5e85afa61e9bfac"
21:07:23.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:23 on dist-test-slave-vhkf"
21:07:23.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.211917 32650 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/wal/instance:
21:07:23.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd1e6f298ab04727a5e85afa61e9bfac"
21:07:23.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:23 on dist-test-slave-vhkf"
21:07:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.227916 32650 fs_manager.cc:696] Time spent creating directory manager: real 0.015s user 0.012s sys 0.004s
21:07:23.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.239789 32671 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:23.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.241504 32650 fs_manager.cc:730] Time spent opening block manager: real 0.008s user 0.003s sys 0.004s
21:07:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.241992 32650 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/wal
21:07:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "dd1e6f298ab04727a5e85afa61e9bfac"
21:07:23.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:23 on dist-test-slave-vhkf"
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.242509 32650 fs_report.cc:389] FS layout report
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/wal
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/wal
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/data/data
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:23.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:23.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.357568 32650 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:23.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.371883 32650 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:23.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.372674 32650 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:23.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.378931 32650 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:07:23.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.388127 32650 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:07:23.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.388427 32650 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:07:23.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.388685 32650 ts_tablet_manager.cc:616] Registered 0 tablets
21:07:23.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:23.388826 32650 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:07:24.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.037426 32650 rpc_server.cc:307] RPC server started. Bound to: 127.31.66.131:44133
21:07:24.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.037789 316 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.66.131:44133 every 8 connection(s)
21:07:24.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.040813 32650 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/data/info.pb
21:07:24.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.047633 32010 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 32650
21:07:24.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.049032 32010 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/ts-2/wal/instance
21:07:24.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:24 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573643, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.131@KRBTEST.COM for kudu/127.31.66.188@KRBTEST.COM
21:07:24.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:24 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573643, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.131@KRBTEST.COM for kudu/127.31.66.189@KRBTEST.COM
21:07:24.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:24 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573643, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.131@KRBTEST.COM for kudu/127.31.66.190@KRBTEST.COM
21:07:24.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.190052 317 heartbeater.cc:344] Connected to a master server at 127.31.66.188:39381
21:07:24.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.190658 317 heartbeater.cc:461] Registering TS with master...
21:07:24.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.192726 317 heartbeater.cc:507] Master 127.31.66.188:39381 requested a full tablet report, sending...
21:07:24.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.199892 32246 ts_manager.cc:194] Registered new tserver with Master: dd1e6f298ab04727a5e85afa61e9bfac (127.31.66.131:44133)
21:07:24.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.207865 318 heartbeater.cc:344] Connected to a master server at 127.31.66.189:33973
21:07:24.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.208336 318 heartbeater.cc:461] Registering TS with master...
21:07:24.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.210121 318 heartbeater.cc:507] Master 127.31.66.189:33973 requested a full tablet report, sending...
21:07:24.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.214986 32158 ts_manager.cc:194] Registered new tserver with Master: dd1e6f298ab04727a5e85afa61e9bfac (127.31.66.131:44133)
21:07:24.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.219493 323 heartbeater.cc:344] Connected to a master server at 127.31.66.190:39107
21:07:24.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.220031 323 heartbeater.cc:461] Registering TS with master...
21:07:24.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.221897 323 heartbeater.cc:507] Master 127.31.66.190:39107 requested a full tablet report, sending...
21:07:24.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.226478 32073 ts_manager.cc:194] Registered new tserver with Master: dd1e6f298ab04727a5e85afa61e9bfac (127.31.66.131:44133)
21:07:24.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.228689 32010 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:07:24.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.228739 32073 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.31.66.131@KRBTEST.COM'} at 127.31.66.131:57301
21:07:24.251 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:24.253 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
21:07:24.264 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
21:07:24.265 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
21:07:24.269 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.31.66.190"
port: 39107
}
bound_http_address {
host: "127.31.66.190"
port: 35807
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.31.66.189"
port: 33973
}
bound_http_address {
host: "127.31.66.189"
port: 44435
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.31.66.188"
port: 39381
}
bound_http_address {
host: "127.31.66.188"
port: 46593
}
}
}
21:07:24.270 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:07:24.271 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.31.66.129"
port: 42691
}
bound_http_address {
host: "127.31.66.129"
port: 32897
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.31.66.130"
port: 42951
}
bound_http_address {
host: "127.31.66.130"
port: 39171
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.31.66.131"
port: 44133
}
bound_http_address {
host: "127.31.66.131"
port: 41903
}
}
}
21:07:24.272 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:07:24.273 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:07:24.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:24 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573644, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:24.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:07:24.319 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:24.322 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc
21:07:24.330 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:07:24.331 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc
21:07:24.332 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:07:24.337 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
21:07:24.340 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:07:24.340 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:07:24.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:24 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573644, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:24.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:07:24.381 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:24.381 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc
21:07:24.383 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:07:24.383 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.31.66.188:39381
21:07:24.385 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:07:24.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.386144 32010 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 32203
21:07:24.416 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:24.416 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.31.66.189:33973
21:07:24.417 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:07:24.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.417577 32010 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 32118
21:07:24.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:24.448 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.31.66.190:39107
21:07:24.448 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:07:24.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.449081 32010 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 32039
21:07:24.490 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:24.490 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.31.66.188:39381
21:07:24.493 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:24.495635 32010 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/logs
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data/info.pb
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:24.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.188:39381
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.188
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46593
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.188
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:24.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:07:24.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:24.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:24.698844 32636 heartbeater.cc:646] Failed to heartbeat to 127.31.66.188:39381 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:24.994415 331 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:24.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:24.995256 331 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:24.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:24.995970 331 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:25.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.037366 331 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:07:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.037739 331 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:07:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.038255 331 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:25.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.038645 331 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.039021 331 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:07:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.039351 331 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:25.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.039572 331 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42783/jwks.json
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:25.110980 331 master_runner.cc:386] Master server non-default flags:
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:25.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.188
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.188:39381
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data/info.pb
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.188
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46593
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/logs
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:25.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:07:25.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:07:25.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:25.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:25.112974 331 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:25.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:25.114341 331 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:25.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.128297 340 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:25.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.148511 32488 heartbeater.cc:646] Failed to heartbeat to 127.31.66.188:39381 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:25.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:25.234570 323 heartbeater.cc:499] Master 127.31.66.190:39107 was elected leader, sending a full tablet report...
21:07:25.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.240612 317 heartbeater.cc:646] Failed to heartbeat to 127.31.66.188:39381 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.31.66.188:39381: connect: Connection refused (error 111)
21:07:26.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.143493 343 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:26.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:25.138615 341 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:26.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:26.677079 342 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1518 milliseconds
21:07:26.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.678241 331 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:26.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.691390 331 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:26.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.703636 331 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:26.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.707875 331 hybrid_clock.cc:648] HybridClock initialized: now 1765573646707651 us; error 218 us; skew 500 ppm
21:07:26.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.714504 331 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:26.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:26 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573646, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.188@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:26.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.752830 331 init.cc:377] Logged in from keytab as kudu/127.31.66.188@KRBTEST.COM (short username kudu)
21:07:26.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.787226 331 webserver.cc:492] Webserver started at http://127.31.66.188:46593/ using document root <none> and password file <none>
21:07:26.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.791270 331 fs_manager.cc:362] Metadata directory not provided
21:07:26.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.792299 331 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:26.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.845011 331 fs_manager.cc:714] Time spent opening directory manager: real 0.036s user 0.021s sys 0.002s
21:07:26.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.891404 366 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:26.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.898962 331 fs_manager.cc:730] Time spent opening block manager: real 0.037s user 0.015s sys 0.009s
21:07:26.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.900075 331 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal
21:07:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "99f382df85844e529eb3e6a120dcb1ee"
21:07:26.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:12 on dist-test-slave-vhkf"
21:07:26.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:26.926124 331 fs_report.cc:389] FS layout report
21:07:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal
21:07:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/wal
21:07:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data/data
21:07:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:26.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:26.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:26.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:26.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:26.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:26.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:27.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.003508 331 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.010847 331 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:27.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.011346 331 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:27.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.104936 331 rpc_server.cc:307] RPC server started. Bound to: 127.31.66.188:39381
21:07:27.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.104992 417 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.66.188:39381 every 8 connection(s)
21:07:27.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.108235 331 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-2/data/info.pb
21:07:27.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.116719 32010 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 331
21:07:27.117 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:27.117 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.31.66.189:33973
21:07:27.118 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.119632 32010 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/logs
21:07:27.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data/info.pb
21:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:27.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.189:33973
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.189
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44435
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.189
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:27.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.122253 418 sys_catalog.cc:263] Verifying existing consensus state
21:07:27.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.128437 418 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: Bootstrap starting.
21:07:27.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.177249 418 log.cc:826] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: Log is configured to *not* fsync() on all Append() calls
21:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.200783 422 negotiation.cc:337] Failed RPC negotiation. Trace:
21:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:27.166252 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.31.66.129:35727 (local address 127.31.66.188:39381)
21:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:27.190291 (+ 24039us) server_negotiation.cc:207] Beginning negotiation
21:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:27.190329 (+ 38us) server_negotiation.cc:400] Waiting for connection header
21:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:27.190389 (+ 60us) server_negotiation.cc:408] Connection header received
21:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:27.195745 (+ 5356us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:27.195795 (+ 50us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:27.196300 (+ 505us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:27.196735 (+ 435us) server_negotiation.cc:227] Negotiated authn=SASL
21:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:27.199236 (+ 2501us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.31.66.129:35727: BlockingRecv error: recv got EOF from 127.31.66.129:35727 (error 108)
21:07:27.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":23337,"thread_start_us":22843,"threads_started":1}
21:07:27.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.207337 418 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:07:27.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.208359 418 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: Bootstrap complete.
21:07:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.230931 418 raft_consensus.cc:359] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:27.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.233458 418 raft_consensus.cc:740] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 99f382df85844e529eb3e6a120dcb1ee, State: Initialized, Role: FOLLOWER
21:07:27.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.234452 418 consensus_queue.cc:260] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.237524 425 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:27.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.238550 425 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: This master's current role is: FOLLOWER
21:07:27.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.239085 418 sys_catalog.cc:565] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: configured and running, proceeding with master startup.
21:07:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.311204 317 heartbeater.cc:344] Connected to a master server at 127.31.66.188:39381
21:07:27.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.324122 439 catalog_manager.cc:1269] Loaded cluster ID: 50892035360048b5a5203f4bbf222c78
21:07:27.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.324525 439 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: loading cluster ID for follower catalog manager: success
21:07:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.333007 439 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: acquiring CA information for follower catalog manager: success
21:07:27.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.339473 439 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:07:27.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.353727 383 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "dd1e6f298ab04727a5e85afa61e9bfac" instance_seqno: 1765573643912675) as {username='kudu', principal='kudu/127.31.66.131@KRBTEST.COM'} at 127.31.66.131:45525; Asking this server to re-register.
21:07:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.312780 331 master_runner.cc:186] Error getting master registration for 127.31.66.190:39107: Network error: Client connection negotiation failed: client connection to 127.31.66.190:39107: connect: Connection refused (error 111),
21:07:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.356009 317 heartbeater.cc:461] Registering TS with master...
21:07:27.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.355885 331 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:07:27.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.360642 383 ts_manager.cc:194] Registered new tserver with Master: dd1e6f298ab04727a5e85afa61e9bfac (127.31.66.131:44133)
21:07:27.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.532303 420 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.532996 420 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:27.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.533733 420 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:27.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:27 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573647, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.129@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:27.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.565920 32370 init.cc:295] Successfully reacquired a new kerberos TGT
21:07:27.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.570241 420 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:07:27.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.570487 420 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:07:27.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.570765 420 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.570969 420 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.571179 420 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:07:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.571378 420 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:27.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.571604 420 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42783/jwks.json
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.615929 420 master_runner.cc:386] Master server non-default flags:
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.189
21:07:27.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:27.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.189:33973
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data/info.pb
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.189
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=44435
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:27.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/logs
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.617833 420 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:27.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.619446 420 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:27.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.635048 447 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:27.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.827265 32636 heartbeater.cc:344] Connected to a master server at 127.31.66.188:39381
21:07:27.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.830238 383 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6af2538206534e97afc6f93aa8201ce3" instance_seqno: 1765573640429925) as {username='kudu', principal='kudu/127.31.66.130@KRBTEST.COM'} at 127.31.66.130:43931; Asking this server to re-register.
21:07:27.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.832031 32636 heartbeater.cc:461] Registering TS with master...
21:07:27.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:27.835168 383 ts_manager.cc:194] Registered new tserver with Master: 6af2538206534e97afc6f93aa8201ce3 (127.31.66.130:42951)
21:07:28.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:28.229038 32488 heartbeater.cc:344] Connected to a master server at 127.31.66.188:39381
21:07:28.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:28.233592 383 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0de102ab31a24754bfc4cec032df3368" instance_seqno: 1765573637904231) as {username='kudu', principal='kudu/127.31.66.129@KRBTEST.COM'} at 127.31.66.129:56519; Asking this server to re-register.
21:07:28.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:28.236419 32488 heartbeater.cc:461] Registering TS with master...
21:07:28.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:28.240844 383 ts_manager.cc:194] Registered new tserver with Master: 0de102ab31a24754bfc4cec032df3368 (127.31.66.129:42691)
21:07:29.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.079695 465 raft_consensus.cc:493] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:07:29.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.085724 465 raft_consensus.cc:515] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:29.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.091250 465 leader_election.cc:290] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 147b937cb1ed40a28e53cd67b23bf42f (127.31.66.190:39107), 06fdd9cd41a0441b93eb0061d60b7672 (127.31.66.189:33973)
21:07:29.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:29.121770 367 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.31.66.190:39107: connect: Connection refused (error 111)
21:07:29.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:29.141054 367 leader_election.cc:336] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 147b937cb1ed40a28e53cd67b23bf42f (127.31.66.190:39107): Network error: Client connection negotiation failed: client connection to 127.31.66.190:39107: connect: Connection refused (error 111)
21:07:29.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:29.146533 367 leader_election.cc:336] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 06fdd9cd41a0441b93eb0061d60b7672 (127.31.66.189:33973): Network error: Client connection negotiation failed: client connection to 127.31.66.189:33973: connect: Connection refused (error 111)
21:07:29.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.149674 367 leader_election.cc:304] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 99f382df85844e529eb3e6a120dcb1ee; no voters: 06fdd9cd41a0441b93eb0061d60b7672, 147b937cb1ed40a28e53cd67b23bf42f
21:07:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.152247 465 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:07:29.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:29.168439 31358 debug-util.cc:398] Leaking SignalData structure 0x7b080009da00 after lost signal to thread 31295
21:07:29.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:29.170257 31358 debug-util.cc:398] Leaking SignalData structure 0x7b080009b040 after lost signal to thread 31361
21:07:29.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:27.636464 448 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:29.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:29.035295 446 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 420
21:07:29.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:29.714246 446 kernel_stack_watchdog.cc:198] Thread 420 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 394ms:
21:07:29.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:07:29.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:07:29.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:07:29.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:07:29.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:29.717279 420 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.079s user 0.002s sys 0.001s
21:07:29.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:29.717664 420 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.079s user 0.002s sys 0.001s
21:07:29.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:29.720945 449 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2078 milliseconds
21:07:29.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:29.722851 451 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:29.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.722925 420 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:29.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.724788 420 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:29.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.743207 420 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:29.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.746905 420 hybrid_clock.cc:648] HybridClock initialized: now 1765573649746835 us; error 64 us; skew 500 ppm
21:07:29.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.748163 420 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:29.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:29 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573649, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.189@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:29.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.757702 420 init.cc:377] Logged in from keytab as kudu/127.31.66.189@KRBTEST.COM (short username kudu)
21:07:29.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.765030 420 webserver.cc:492] Webserver started at http://127.31.66.189:44435/ using document root <none> and password file <none>
21:07:29.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.766597 420 fs_manager.cc:362] Metadata directory not provided
21:07:29.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.766943 420 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:29.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.792186 420 fs_manager.cc:714] Time spent opening directory manager: real 0.020s user 0.012s sys 0.008s
21:07:29.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.811195 476 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:29.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.813073 420 fs_manager.cc:730] Time spent opening block manager: real 0.017s user 0.007s sys 0.009s
21:07:29.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.813817 420 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal
21:07:29.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "06fdd9cd41a0441b93eb0061d60b7672"
21:07:29.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:10 on dist-test-slave-vhkf"
21:07:29.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.817248 420 fs_report.cc:389] FS layout report
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/wal
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data/data
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:29.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:29.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:29.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:29.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:29.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.905733 420 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.918025 420 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:29.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:29.918692 420 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:30.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:30 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573650, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.130@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.045540 32518 init.cc:295] Successfully reacquired a new kerberos TGT
21:07:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.160837 420 rpc_server.cc:307] RPC server started. Bound to: 127.31.66.189:33973
21:07:30.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.160892 528 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.66.189:33973 every 8 connection(s)
21:07:30.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.164140 420 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-1/data/info.pb
21:07:30.174 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:30.174 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.31.66.190:39107
21:07:30.175 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:07:30.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.172374 32010 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 420
21:07:30.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.179298 32010 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/logs
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data/info.pb
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:30.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.190:39107
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.190
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35807
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.190
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:07:30.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:30.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.208943 529 sys_catalog.cc:263] Verifying existing consensus state
21:07:30.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.220173 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: Bootstrap starting.
21:07:30.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:30 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573647, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.129@KRBTEST.COM for kudu/127.31.66.189@KRBTEST.COM
21:07:30.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.331104 32489 heartbeater.cc:344] Connected to a master server at 127.31.66.189:33973
21:07:30.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.386267 529 log.cc:826] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: Log is configured to *not* fsync() on all Append() calls
21:07:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.433070 318 heartbeater.cc:344] Connected to a master server at 127.31.66.189:33973
21:07:30.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.459110 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:07:30.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.460681 529 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: Bootstrap complete.
21:07:30.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.497398 529 raft_consensus.cc:359] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:30.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.502177 529 raft_consensus.cc:740] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 06fdd9cd41a0441b93eb0061d60b7672, State: Initialized, Role: FOLLOWER
21:07:30.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.503978 529 consensus_queue.cc:260] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.514441 540 sys_catalog.cc:455] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:30.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.515349 540 sys_catalog.cc:458] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: This master's current role is: FOLLOWER
21:07:30.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.517273 529 sys_catalog.cc:565] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: configured and running, proceeding with master startup.
21:07:30.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.637355 551 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:07:30.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.645267 420 master_runner.cc:186] Error getting master registration for 127.31.66.190:39107: Network error: Client connection negotiation failed: client connection to 127.31.66.190:39107: connect: Connection refused (error 111),
21:07:30.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.676566 420 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:07:30.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.784808 531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:30.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.785773 531 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:30.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.788273 531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:30.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:30 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573650, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.130@KRBTEST.COM for kudu/127.31.66.189@KRBTEST.COM
21:07:30.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.860926 531 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:07:30.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.861325 531 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:07:30.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.861864 531 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:30.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.862254 531 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:30.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.862632 531 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:07:30.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.863011 531 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:30.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.863291 531 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:42783/jwks.json
21:07:30.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.871042 32637 heartbeater.cc:344] Connected to a master server at 127.31.66.189:33973
21:07:30.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.879701 494 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6af2538206534e97afc6f93aa8201ce3" instance_seqno: 1765573640429925) as {username='kudu', principal='kudu/127.31.66.130@KRBTEST.COM'} at 127.31.66.130:42439; Asking this server to re-register.
21:07:30.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.881748 32637 heartbeater.cc:461] Registering TS with master...
21:07:30.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.887403 493 ts_manager.cc:194] Registered new tserver with Master: 6af2538206534e97afc6f93aa8201ce3 (127.31.66.130:42951)
21:07:30.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.890053 558 raft_consensus.cc:493] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:07:30.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.890901 558 raft_consensus.cc:515] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:30.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.903561 558 leader_election.cc:290] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 147b937cb1ed40a28e53cd67b23bf42f (127.31.66.190:39107), 06fdd9cd41a0441b93eb0061d60b7672 (127.31.66.189:33973)
21:07:30.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.933521 367 leader_election.cc:336] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 147b937cb1ed40a28e53cd67b23bf42f (127.31.66.190:39107): Network error: Client connection negotiation failed: client connection to 127.31.66.190:39107: connect: Connection refused (error 111)
21:07:30.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:30 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573646, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.188@KRBTEST.COM for kudu/127.31.66.189@KRBTEST.COM
21:07:30.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.955776 504 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99f382df85844e529eb3e6a120dcb1ee" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "06fdd9cd41a0441b93eb0061d60b7672" is_pre_election: true
21:07:30.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.956916 504 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 99f382df85844e529eb3e6a120dcb1ee in term 1.
21:07:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.958771 367 leader_election.cc:304] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 06fdd9cd41a0441b93eb0061d60b7672, 99f382df85844e529eb3e6a120dcb1ee; no voters: 147b937cb1ed40a28e53cd67b23bf42f
21:07:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.959889 558 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Leader pre-election won for term 2
21:07:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.960269 558 raft_consensus.cc:493] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:07:30.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.960610 558 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 1 FOLLOWER]: Advancing to term 2
21:07:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.980017 531 master_runner.cc:386] Master server non-default flags:
21:07:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.31.66.148:36447
21:07:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data
21:07:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal
21:07:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:07:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:30.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.31.66.190:39107,127.31.66.189:33973,127.31.66.188:39381
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/krb5kdc/kudu.keytab
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.31.66.190
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.31.66.190:39107
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:42783/jwks.json
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data/info.pb
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.31.66.190
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=35807
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:30.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/logs
21:07:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 468897755969d686fb08386f42f7835685e7953a
21:07:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 12 Dec 2025 20:43:23 UTC on 5fd53c4cbb9d
21:07:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9485
21:07:30.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:30.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.982070 531 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:30.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.983532 531 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:30.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:30.986405 558 raft_consensus.cc:515] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:31.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:30.994812 367 leader_election.cc:336] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 147b937cb1ed40a28e53cd67b23bf42f (127.31.66.190:39107): Network error: Client connection negotiation failed: client connection to 127.31.66.190:39107: connect: Connection refused (error 111)
21:07:31.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.002776 504 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "99f382df85844e529eb3e6a120dcb1ee" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "06fdd9cd41a0441b93eb0061d60b7672"
21:07:31.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.003489 504 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 1 FOLLOWER]: Advancing to term 2
21:07:31.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.012982 558 leader_election.cc:290] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [CANDIDATE]: Term 2 election: Requested vote from peers 147b937cb1ed40a28e53cd67b23bf42f (127.31.66.190:39107), 06fdd9cd41a0441b93eb0061d60b7672 (127.31.66.189:33973)
21:07:31.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.016937 504 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 99f382df85844e529eb3e6a120dcb1ee in term 2.
21:07:31.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.020442 367 leader_election.cc:304] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 06fdd9cd41a0441b93eb0061d60b7672, 99f382df85844e529eb3e6a120dcb1ee; no voters: 147b937cb1ed40a28e53cd67b23bf42f
21:07:31.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.023260 558 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 2 FOLLOWER]: Leader election won for term 2
21:07:31.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.049429 558 raft_consensus.cc:697] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [term 2 LEADER]: Becoming Leader. State: Replica: 99f382df85844e529eb3e6a120dcb1ee, State: Running, Role: LEADER
21:07:31.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.050761 558 consensus_queue.cc:237] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:31.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.075475 563 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99f382df85844e529eb3e6a120dcb1ee. Latest consensus state: current_term: 2 leader_uuid: "99f382df85844e529eb3e6a120dcb1ee" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:31.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.076542 563 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: This master's current role is: LEADER
21:07:31.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.087672 566 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:07:31.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.104934 566 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:07:31.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:31.100003 565 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:31.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.109370 566 catalog_manager.cc:1269] Loaded cluster ID: 50892035360048b5a5203f4bbf222c78
21:07:31.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.109632 566 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:07:31.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.115235 566 catalog_manager.cc:1514] Loading token signing keys...
21:07:31.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.120412 566 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee: Loaded TSK: 0
21:07:31.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.121896 566 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:07:31.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.338791 494 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0de102ab31a24754bfc4cec032df3368" instance_seqno: 1765573637904231) as {username='kudu', principal='kudu/127.31.66.129@KRBTEST.COM'} at 127.31.66.129:39467; Asking this server to re-register.
21:07:31.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.342247 32489 heartbeater.cc:461] Registering TS with master...
21:07:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.346362 494 ts_manager.cc:194] Registered new tserver with Master: 0de102ab31a24754bfc4cec032df3368 (127.31.66.129:42691)
21:07:31.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.442726 494 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "dd1e6f298ab04727a5e85afa61e9bfac" instance_seqno: 1765573643912675) as {username='kudu', principal='kudu/127.31.66.131@KRBTEST.COM'} at 127.31.66.131:42319; Asking this server to re-register.
21:07:31.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.445832 318 heartbeater.cc:461] Registering TS with master...
21:07:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.451546 494 ts_manager.cc:194] Registered new tserver with Master: dd1e6f298ab04727a5e85afa61e9bfac (127.31.66.131:44133)
21:07:31.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.559252 504 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [term 2 FOLLOWER]: Refusing update from remote peer 99f382df85844e529eb3e6a120dcb1ee: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
21:07:31.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.562017 563 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [LEADER]: Connected to new peer: Peer: permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
21:07:31.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:31.623911 367 consensus_peers.cc:597] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee -> Peer 147b937cb1ed40a28e53cd67b23bf42f (127.31.66.190:39107): Couldn't send request to peer 147b937cb1ed40a28e53cd67b23bf42f. Status: Network error: Client connection negotiation failed: client connection to 127.31.66.190:39107: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:07:31.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.637789 574 sys_catalog.cc:455] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99f382df85844e529eb3e6a120dcb1ee. Latest consensus state: current_term: 2 leader_uuid: "99f382df85844e529eb3e6a120dcb1ee" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:31.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.645398 574 sys_catalog.cc:458] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: This master's current role is: FOLLOWER
21:07:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.650560 563 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "99f382df85844e529eb3e6a120dcb1ee" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.651635 563 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: This master's current role is: LEADER
21:07:31.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.663688 551 catalog_manager.cc:1269] Loaded cluster ID: 50892035360048b5a5203f4bbf222c78
21:07:31.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.674836 574 sys_catalog.cc:455] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "99f382df85844e529eb3e6a120dcb1ee" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:31.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.675877 574 sys_catalog.cc:458] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672 [sys.catalog]: This master's current role is: FOLLOWER
21:07:31.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.674870 551 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: loading cluster ID for follower catalog manager: success
21:07:31.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.696141 551 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: acquiring CA information for follower catalog manager: success
21:07:31.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:31.714916 551 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 06fdd9cd41a0441b93eb0061d60b7672: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:07:32.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.257576 32488 heartbeater.cc:499] Master 127.31.66.188:39381 was elected leader, sending a full tablet report...
21:07:32.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.360252 331 master_runner.cc:186] Error getting master registration for 127.31.66.190:39107: Network error: Client connection negotiation failed: client connection to 127.31.66.190:39107: connect: Connection refused (error 111),
21:07:32.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.360940 331 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:07:32.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.391732 317 heartbeater.cc:499] Master 127.31.66.188:39381 was elected leader, sending a full tablet report...
21:07:32.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.856137 32636 heartbeater.cc:499] Master 127.31.66.188:39381 was elected leader, sending a full tablet report...
21:07:32.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:32.466636 562 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 531
21:07:32.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:31.110656 567 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:32.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:32.945312 562 kernel_stack_watchdog.cc:198] Thread 531 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 359ms:
21:07:32.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:07:32.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:07:32.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:07:32.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:07:32.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:32.946229 531 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.841s user 0.565s sys 0.928s
21:07:32.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:32.946697 531 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.842s user 0.565s sys 0.928s
21:07:32.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.953992 531 server_base.cc:1047] running on GCE node
21:07:32.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:32.954098 591 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:32.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.955725 531 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:32.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.958912 531 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:32.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.960400 531 hybrid_clock.cc:648] HybridClock initialized: now 1765573652960321 us; error 76 us; skew 500 ppm
21:07:32.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.961625 531 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:32.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:32 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573652, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.190@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.970855 531 init.cc:377] Logged in from keytab as kudu/127.31.66.190@KRBTEST.COM (short username kudu)
21:07:32.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.977473 531 webserver.cc:492] Webserver started at http://127.31.66.190:35807/ using document root <none> and password file <none>
21:07:32.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.978626 531 fs_manager.cc:362] Metadata directory not provided
21:07:32.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.978969 531 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:32.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.988451 531 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.003s sys 0.003s
21:07:32.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.993850 597 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:32.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.995288 531 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.002s sys 0.002s
21:07:32.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.995723 531 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data,/tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal
21:07:32.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "147b937cb1ed40a28e53cd67b23bf42f"
21:07:32.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-12 21:07:07 on dist-test-slave-vhkf"
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:32.998729 531 fs_report.cc:389] FS layout report
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/wal
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data/data
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:32.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:33.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.112785 531 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:33.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.121693 531 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:33.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.122315 531 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:33.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:33 dist-test-slave-vhkf krb5kdc[32022](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573653, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.131@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:33.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.195163 32667 init.cc:295] Successfully reacquired a new kerberos TGT
21:07:33.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.240254 531 rpc_server.cc:307] RPC server started. Bound to: 127.31.66.190:39107
21:07:33.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.240387 650 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.31.66.190:39107 every 8 connection(s)
21:07:33.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.243674 531 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskGYhehh/test-tmp/mini-kudu-cluster620488566575255660/master-0/data/info.pb
21:07:33.246 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:33.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.245190 32010 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu as pid 531
21:07:33.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.260082 651 sys_catalog.cc:263] Verifying existing consensus state
21:07:33.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.268031 651 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: Bootstrap starting.
21:07:33.311 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.31.66.188:39381(127.31.66.188:39381)] connecting to peer
21:07:33.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.336171 651 log.cc:826] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: Log is configured to *not* fsync() on all Append() calls
21:07:33.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:33 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573647, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.129@KRBTEST.COM for kudu/127.31.66.190@KRBTEST.COM
21:07:33.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.365707 651 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:07:33.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.366811 651 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: Bootstrap complete.
21:07:33.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.372231 32490 heartbeater.cc:344] Connected to a master server at 127.31.66.190:39107
21:07:33.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.399379 651 raft_consensus.cc:359] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:33.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.402817 651 raft_consensus.cc:740] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 147b937cb1ed40a28e53cd67b23bf42f, State: Initialized, Role: FOLLOWER
21:07:33.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.403961 651 consensus_queue.cc:260] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } }
21:07:33.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.411844 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:33.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.413189 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: This master's current role is: FOLLOWER
21:07:33.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.414021 651 sys_catalog.cc:565] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: configured and running, proceeding with master startup.
21:07:33.422 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.31.66.189:33973(127.31.66.189:33973)] connecting to peer
21:07:33.426 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.31.66.190:39107(127.31.66.190:39107)] connecting to peer
21:07:33.503 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.31.66.189:33973(127.31.66.189:33973)] Successfully connected to peer
21:07:33.514 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.31.66.188:39381(127.31.66.188:39381)] Successfully connected to peer
21:07:33.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:33 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573653, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.131@KRBTEST.COM for kudu/127.31.66.190@KRBTEST.COM
21:07:33.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.554617 323 heartbeater.cc:344] Connected to a master server at 127.31.66.190:39107
21:07:33.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.555357 323 heartbeater.cc:499] Master 127.31.66.190:39107 was elected leader, sending a full tablet report...
21:07:33.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.604579 616 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "dd1e6f298ab04727a5e85afa61e9bfac" instance_seqno: 1765573643912675) as {username='kudu', principal='kudu/127.31.66.131@KRBTEST.COM'} at 127.31.66.131:49035; Asking this server to re-register.
21:07:33.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.610539 323 heartbeater.cc:461] Registering TS with master...
21:07:33.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.627628 616 ts_manager.cc:194] Registered new tserver with Master: dd1e6f298ab04727a5e85afa61e9bfac (127.31.66.131:44133)
21:07:33.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.647897 674 catalog_manager.cc:1269] Loaded cluster ID: 50892035360048b5a5203f4bbf222c78
21:07:33.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.648278 674 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: loading cluster ID for follower catalog manager: success
21:07:33.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:33 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573652, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.190@KRBTEST.COM for kudu/127.31.66.190@KRBTEST.COM
21:07:33.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.690296 674 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: acquiring CA information for follower catalog manager: success
21:07:33.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.698562 674 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:07:33.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 12 21:07:33 dist-test-slave-vhkf krb5kdc[32022](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765573646, etypes {rep=17 tkt=17 ses=17}, kudu/127.31.66.188@KRBTEST.COM for kudu/127.31.66.190@KRBTEST.COM
21:07:33.729 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.31.66.190:39107(127.31.66.190:39107)] Successfully connected to peer
21:07:33.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.736812 626 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [term 1 FOLLOWER]: Advancing to term 2
21:07:33.801 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
21:07:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.824306 657 sys_catalog.cc:455] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99f382df85844e529eb3e6a120dcb1ee. Latest consensus state: current_term: 2 leader_uuid: "99f382df85844e529eb3e6a120dcb1ee" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.825260 657 sys_catalog.cc:458] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: This master's current role is: FOLLOWER
21:07:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.828590 683 sys_catalog.cc:455] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "99f382df85844e529eb3e6a120dcb1ee" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:33.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.829793 683 sys_catalog.cc:458] T 00000000000000000000000000000000 P 147b937cb1ed40a28e53cd67b23bf42f [sys.catalog]: This master's current role is: FOLLOWER
21:07:33.857 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.31.66.189
21:07:33.858 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.31.66.188
21:07:33.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.979430 32638 heartbeater.cc:344] Connected to a master server at 127.31.66.190:39107
21:07:33.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.983824 616 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "6af2538206534e97afc6f93aa8201ce3" instance_seqno: 1765573640429925) as {username='kudu', principal='kudu/127.31.66.130@KRBTEST.COM'} at 127.31.66.130:53549; Asking this server to re-register.
21:07:33.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:33.986640 32638 heartbeater.cc:461] Registering TS with master...
21:07:34.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.001293 616 ts_manager.cc:194] Registered new tserver with Master: 6af2538206534e97afc6f93aa8201ce3 (127.31.66.130:42951)
21:07:34.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.034502 563 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "99f382df85844e529eb3e6a120dcb1ee" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "147b937cb1ed40a28e53cd67b23bf42f" member_type: VOTER last_known_addr { host: "127.31.66.190" port: 39107 } } peers { permanent_uuid: "06fdd9cd41a0441b93eb0061d60b7672" member_type: VOTER last_known_addr { host: "127.31.66.189" port: 33973 } } peers { permanent_uuid: "99f382df85844e529eb3e6a120dcb1ee" member_type: VOTER last_known_addr { host: "127.31.66.188" port: 39381 } } }
21:07:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.035809 563 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99f382df85844e529eb3e6a120dcb1ee [sys.catalog]: This master's current role is: LEADER
21:07:34.113 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.66.190:39107(127.31.66.190:39107)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:07:34.123 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.66.190:39107: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
21:07:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:34.127044 671 negotiation.cc:337] Failed RPC negotiation. Trace:
21:07:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.509167 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:34574 (local address 127.31.66.190:39107)
21:07:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.532311 (+ 23144us) server_negotiation.cc:207] Beginning negotiation
21:07:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.532364 (+ 53us) server_negotiation.cc:400] Waiting for connection header
21:07:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.730428 (+198064us) server_negotiation.cc:408] Connection header received
21:07:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.732647 (+ 2219us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:07:34.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.732691 (+ 44us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:07:34.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.733039 (+ 348us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:07:34.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.733303 (+ 264us) server_negotiation.cc:227] Negotiated authn=SASL
21:07:34.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:34.124448 (+391145us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:34574: BlockingRecv error: recv got EOF from 127.0.0.1:34574 (error 108)
21:07:34.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":21562,"thread_start_us":23985,"threads_started":1}
21:07:34.134 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.31.66.190:39107(127.31.66.190:39107)] handling channelInactive
21:07:34.293 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.31.66.188:39381(127.31.66.188:39381)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:07:34.294 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.31.66.188:39381: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:07:34.296 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.31.66.189:33973(127.31.66.189:33973)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:34.301751 670 negotiation.cc:337] Failed RPC negotiation. Trace:
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.507682 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:41398 (local address 127.31.66.188:39381)
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.514477 (+ 6795us) server_negotiation.cc:207] Beginning negotiation
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.514542 (+ 65us) server_negotiation.cc:400] Waiting for connection header
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.594072 (+ 79530us) server_negotiation.cc:408] Connection header received
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.731201 (+137129us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.731245 (+ 44us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.731589 (+ 344us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.731847 (+ 258us) server_negotiation.cc:227] Negotiated authn=SASL
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:34.080171 (+348324us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:34.082556 (+ 2385us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:34.268499 (+185943us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:07:34.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:34.269432 (+ 933us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
21:07:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:34.299287 (+ 29855us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:41398: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:41398): Cannot send after transport endpoint shutdown (error 108)
21:07:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5894,"thread_start_us":5539,"threads_started":1}
21:07:34.313 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.31.66.189:33973: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:07:34.332 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.31.66.188:39381(127.31.66.188:39381)] handling channelInactive
21:07:34.334 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1078), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.66.188:39381, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.66.189:33973, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.66.190:39107, [858ms] Sub RPC ConnectToMaster: received response from server master-127.31.66.190:39107: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [1033ms] Sub RPC ConnectToMaster: received response from server master-127.31.66.188:39381: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1036ms] Sub RPC ConnectToMaster: received response from server master-127.31.66.189:33973: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.31.66.188:39381,127.31.66.189:33973,127.31.66.190:39107). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.31.66.188:39381,127.31.66.189:33973,127.31.66.190:39107). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21:07:34.357 [DEBUG - kudu-nio-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1095), Traces: [0ms] refreshing cache from master, [38ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.66.188:39381, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.66.189:33973, [166ms] Sub RPC ConnectToMaster: sending RPC to server master-127.31.66.190:39107, [858ms] Sub RPC ConnectToMaster: received response from server master-127.31.66.190:39107: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [1033ms] Sub RPC ConnectToMaster: received response from server master-127.31.66.188:39381: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1036ms] Sub RPC ConnectToMaster: received response from server master-127.31.66.189:33973: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
21:07:34.359 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.31.66.189:33973(127.31.66.189:33973)] handling channelInactive
21:07:34.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:34.367220 668 negotiation.cc:337] Failed RPC negotiation. Trace:
21:07:34.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.499472 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:49542 (local address 127.31.66.189:33973)
21:07:34.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.517724 (+ 18252us) server_negotiation.cc:207] Beginning negotiation
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.517779 (+ 55us) server_negotiation.cc:400] Waiting for connection header
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.600929 (+ 83150us) server_negotiation.cc:408] Connection header received
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.725501 (+124572us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.725535 (+ 34us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.725808 (+ 273us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:33.726032 (+ 224us) server_negotiation.cc:227] Negotiated authn=SASL
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:34.067453 (+341421us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:34.071525 (+ 4072us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:34.271001 (+199476us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:34.271760 (+ 759us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1212 21:07:34.363310 (+ 91550us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:49542: BlockingRecv error: failed to read from TLS socket (remote: unknown): Cannot send after transport endpoint shutdown (error 108)
21:07:34.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":16917,"thread_start_us":16289,"threads_started":1}
21:07:34.370 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.31.66.189:33973(127.31.66.189:33973)] disconnecting while in state TERMINATED
21:07:34.373 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.31.66.188:39381(127.31.66.188:39381)] disconnecting while in state TERMINATED
21:07:34.375 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.31.66.190:39107(127.31.66.190:39107)] disconnecting while in state TERMINATED
21:07:34.379 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:07:34.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.380219 616 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "0de102ab31a24754bfc4cec032df3368" instance_seqno: 1765573637904231) as {username='kudu', principal='kudu/127.31.66.129@KRBTEST.COM'} at 127.31.66.129:51707; Asking this server to re-register.
21:07:34.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.382853 32490 heartbeater.cc:461] Registering TS with master...
21:07:34.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.388453 616 ts_manager.cc:194] Registered new tserver with Master: 0de102ab31a24754bfc4cec032df3368 (127.31.66.129:42691)
21:07:34.390 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:07:34.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.427459 32010 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 32351
21:07:34.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.464524 32010 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 32502
21:07:34.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.499233 32010 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 32650
21:07:34.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.528594 32010 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 531
21:07:34.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.555845 32010 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 420
21:07:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:34.581622 367 connection.cc:537] client connection to 127.31.66.189:33973 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:07:34.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251212 21:07:34.582083 367 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107) [suppressed 7 similar messages]
21:07:34.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:34.582798 32010 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 331
21:07:35.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-12T21:07:35Z chronyd exiting
21:07:36.841 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:07:36.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:36.843010 31282 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 31563
21:07:36.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:36.870334 31282 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 31725
21:07:36.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:36.898218 31282 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 31867
21:07:36.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:36.925021 31282 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 31294
21:07:36.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:36.962049 31282 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 31364
21:07:36.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251212 21:07:36.994164 31282 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskGYhehh/build/tsan/bin/kudu with pid 31439
21:07:38.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-12T21:07:38Z chronyd exiting
21:07:39.206 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.31.66.188:39381,127.31.66.189:33973,127.31.66.190:39107). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.31.66.188:39381,127.31.66.189:33973,127.31.66.190:39107). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more