21:06:30.500 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
21:06:30.512 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNTuEsu/test-tmp
21:06:30.515 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:06:30.518 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNTuEsu/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:06:30.747 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
21:06:30.903 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:06:30.905 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:06:30.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:30.907982 30254 mini_oidc.cc:150] Starting JWKS server
21:06:30.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:30.911231 30254 webserver.cc:492] Webserver started at http://127.0.0.1:38293/ using document root <none> and password file <none>
21:06:30.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:30.911736 30254 mini_oidc.cc:186] Starting OIDC Discovery server
21:06:30.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:30.913650 30254 webserver.cc:492] Webserver started at http://127.0.0.1:46453/ using document root <none> and password file <none>
21:06:30.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-14T21:06:30Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:06:30.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-14T21:06:30Z Disabled control of system clock
21:06:30.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:30.966538 30254 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:30.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:30.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/wal
21:06:30.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/data
21:06:30.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:30.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:30.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/logs
21:06:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/data/info.pb
21:06:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:30.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:06:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:30.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:30.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:30.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.190:32895
21:06:30.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.190
21:06:30.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:30.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:30.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:30.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:30.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:30.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38293/jwks.json with env {}
21:06:31.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:31.245174 30266 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:31.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:31.245695 30266 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:31.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:31.246343 30266 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:31.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:31.277482 30266 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:06:31.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:31.277763 30266 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:31.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:31.277951 30266 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:06:31.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:31.278134 30266 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:06:31.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:31.278416 30266 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:06:31.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:31.278671 30266 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38293/jwks.json
21:06:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:31.313055 30266 master_runner.cc:386] Master server non-default flags:
21:06:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:31.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/data
21:06:31.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/wal
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.190:32895
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38293/jwks.json
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:31.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/data/info.pb
21:06:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.190
21:06:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/logs
21:06:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:31.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:06:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:06:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:06:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:06:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:31.314529 30266 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:31.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:31.316638 30266 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:31.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:31.329329 30272 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:32.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:31.329838 30273 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:32.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:32.731323 30271 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec80 after lost signal to thread 30266
21:06:32.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:32.854038 30266 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.525s user 0.603s sys 0.922s
21:06:32.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:32.856389 30275 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:32.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:32.856761 30274 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1525 milliseconds
21:06:32.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:32.857126 30266 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.528s user 0.603s sys 0.922s
21:06:32.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.857424 30266 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:06:32.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.860541 30266 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:32.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.862758 30266 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:32.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.864069 30266 hybrid_clock.cc:648] HybridClock initialized: now 1765746392864040 us; error 35 us; skew 500 ppm
21:06:32.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.864892 30266 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:32.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.870399 30266 webserver.cc:492] Webserver started at http://127.29.139.190:40747/ using document root <none> and password file <none>
21:06:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.871218 30266 fs_manager.cc:362] Metadata directory not provided
21:06:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.871397 30266 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:32.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.871830 30266 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:32.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.875921 30266 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/data/instance:
21:06:32.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91288c01a71f43deaf02f5b945a32fe2"
21:06:32.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:32 on dist-test-slave-x0xn"
21:06:32.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.876997 30266 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/wal/instance:
21:06:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91288c01a71f43deaf02f5b945a32fe2"
21:06:32.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:32 on dist-test-slave-x0xn"
21:06:32.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.883563 30266 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.001s
21:06:32.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.888826 30282 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.889801 30266 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
21:06:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.890071 30266 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/wal
21:06:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "91288c01a71f43deaf02f5b945a32fe2"
21:06:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:32 on dist-test-slave-x0xn"
21:06:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.890379 30266 fs_report.cc:389] FS layout report
21:06:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/wal
21:06:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/wal
21:06:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/data/data
21:06:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:32.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:32.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:32.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.950908 30266 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.952292 30266 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:32.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:32.952677 30266 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:33.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.017583 30266 rpc_server.cc:307] RPC server started. Bound to: 127.29.139.190:32895
21:06:33.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.017668 30334 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.139.190:32895 every 8 connection(s)
21:06:33.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.020112 30266 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/data/info.pb
21:06:33.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.025592 30335 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:06:33.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.027573 30254 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 30266
21:06:33.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.027983 30254 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-0/wal/instance
21:06:33.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.031278 30254 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:33.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:33.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/wal
21:06:33.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/data
21:06:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/logs
21:06:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/data/info.pb
21:06:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:33.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:33.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:33.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:33.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:33.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:33.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:33.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:33.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:06:33.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:33.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:33.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:33.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.189:45383
21:06:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.189
21:06:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:33.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:33.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:33.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38293/jwks.json with env {}
21:06:33.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.037128 30335 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } has no permanent_uuid. Determining permanent_uuid...
21:06:33.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.064528 30335 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } has no permanent_uuid. Determining permanent_uuid...
21:06:33.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.066862 30283 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.139.189:45383: connect: Connection refused (error 111)
21:06:33.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.070770 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.189:45383: Network error: Client connection negotiation failed: client connection to 127.29.139.189:45383: connect: Connection refused (error 111)
21:06:33.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.120204 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } attempt: 1
21:06:33.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.123834 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.189:45383: Network error: Client connection negotiation failed: client connection to 127.29.139.189:45383: connect: Connection refused (error 111)
21:06:33.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.192265 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } attempt: 2
21:06:33.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.195937 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.189:45383: Network error: Client connection negotiation failed: client connection to 127.29.139.189:45383: connect: Connection refused (error 111)
21:06:33.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.287402 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } attempt: 3
21:06:33.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.291242 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.189:45383: Network error: Client connection negotiation failed: client connection to 127.29.139.189:45383: connect: Connection refused (error 111)
21:06:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.328543 30337 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.329053 30337 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:33.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.329495 30337 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.359323 30337 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:06:33.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.359575 30337 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:33.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.359839 30337 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:06:33.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.360044 30337 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:06:33.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.360265 30337 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:06:33.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.360406 30337 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38293/jwks.json
21:06:33.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.394300 30337 master_runner.cc:386] Master server non-default flags:
21:06:33.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:33.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/data
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/wal
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.189:45383
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38293/jwks.json
21:06:33.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:33.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:33.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/data/info.pb
21:06:33.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.189
21:06:33.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:33.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:33.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/logs
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:06:33.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:06:33.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:33.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.395536 30337 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:33.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.397058 30337 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:33.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.408573 30346 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:33.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.434600 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } attempt: 4
21:06:33.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.448163 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.189:45383: Network error: Client connection negotiation failed: client connection to 127.29.139.189:45383: connect: Connection refused (error 111)
21:06:33.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:33.747653 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } attempt: 5
21:06:33.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.754719 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.189:45383: Network error: Client connection negotiation failed: client connection to 127.29.139.189:45383: connect: Connection refused (error 111)
21:06:34.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.302207 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } attempt: 6
21:06:34.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:34.326987 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.189:45383: Network error: Client connection negotiation failed: client connection to 127.29.139.189:45383: connect: Connection refused (error 111)
21:06:34.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:33.408694 30347 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:34.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:34.641671 30348 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1228 milliseconds
21:06:34.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:34.643476 30349 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:34.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:34.645522 30337 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.236s user 0.000s sys 0.008s
21:06:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:34.645785 30337 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.237s user 0.000s sys 0.008s
21:06:34.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.646067 30337 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:06:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.647193 30337 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.649683 30337 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:34.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.651023 30337 hybrid_clock.cc:648] HybridClock initialized: now 1765746394650976 us; error 56 us; skew 500 ppm
21:06:34.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.651846 30337 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:34.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.658532 30337 webserver.cc:492] Webserver started at http://127.29.139.189:37393/ using document root <none> and password file <none>
21:06:34.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.659463 30337 fs_manager.cc:362] Metadata directory not provided
21:06:34.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.659677 30337 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:34.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.660218 30337 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:34.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.664386 30337 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/data/instance:
21:06:34.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8fef5978fb04a3e83dd90a7a9285656"
21:06:34.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:34 on dist-test-slave-x0xn"
21:06:34.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.665535 30337 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/wal/instance:
21:06:34.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8fef5978fb04a3e83dd90a7a9285656"
21:06:34.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:34 on dist-test-slave-x0xn"
21:06:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.673143 30337 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.005s
21:06:34.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.678810 30358 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.679996 30337 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
21:06:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.680405 30337 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/wal
21:06:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a8fef5978fb04a3e83dd90a7a9285656"
21:06:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:34 on dist-test-slave-x0xn"
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.680848 30337 fs_report.cc:389] FS layout report
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/wal
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/wal
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/data/data
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:34.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:34.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:34.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.767714 30337 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:34.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.769088 30337 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:34.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.769443 30337 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.832967 30337 rpc_server.cc:307] RPC server started. Bound to: 127.29.139.189:45383
21:06:34.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.833055 30410 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.139.189:45383 every 8 connection(s)
21:06:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.835428 30337 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/data/info.pb
21:06:34.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.840858 30411 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:06:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.843504 30254 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 30337
21:06:34.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.843830 30254 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-1/wal/instance
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.845736 30254 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/wal
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/data
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/logs
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/data/info.pb
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:34.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.188:37723
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.188
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:34.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:34.848 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:34.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38293/jwks.json with env {}
21:06:34.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.852768 30411 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } has no permanent_uuid. Determining permanent_uuid...
21:06:34.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.874717 30411 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } has no permanent_uuid. Determining permanent_uuid...
21:06:34.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.887439 30411 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } has no permanent_uuid. Determining permanent_uuid...
21:06:34.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:34.889736 30360 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:34.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:34.893389 30411 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:34.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:34.942888 30411 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 1
21:06:34.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:34.946925 30411 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:35.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.015364 30411 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 2
21:06:35.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.019776 30411 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:35.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.111205 30411 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 3
21:06:35.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.114833 30411 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:35.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.141482 30413 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:35.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.141986 30413 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:35.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.142427 30413 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:35.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.172390 30413 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:06:35.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.172648 30413 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:35.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.172834 30413 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:06:35.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.173048 30413 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:06:35.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.173247 30413 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:06:35.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.173391 30413 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38293/jwks.json
21:06:35.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.206611 30413 master_runner.cc:386] Master server non-default flags:
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/data
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/wal
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.188:37723
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38293/jwks.json
21:06:35.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/data/info.pb
21:06:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.188
21:06:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:35.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/logs
21:06:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:06:35.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:06:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:06:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:06:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.208096 30413 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:35.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.210107 30413 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:35.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.222397 30423 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:35.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.258258 30411 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 4
21:06:35.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.273350 30411 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:35.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.387461 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } attempt: 7
21:06:35.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.434682 30335 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } has no permanent_uuid. Determining permanent_uuid...
21:06:35.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.451743 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:35.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.510222 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 1
21:06:35.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.516450 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:35.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.572824 30411 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 5
21:06:35.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.579226 30411 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.597919 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 2
21:06:35.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.603909 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:35.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.689383 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 3
21:06:35.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.695484 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:35.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:35.835959 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 4
21:06:35.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.852332 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:36.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.126643 30411 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 6
21:06:36.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.135819 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 5
21:06:36.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:36.142019 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:36.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:36.160211 30411 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:36.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:35.223455 30424 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:36.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:36.540316 30425 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1316 milliseconds
21:06:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:36.541193 30426 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:36.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:36.545078 30413 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.322s user 0.479s sys 0.829s
21:06:36.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:36.545680 30413 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.323s user 0.480s sys 0.830s
21:06:36.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.546156 30413 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:06:36.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.547766 30413 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:36.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.559387 30413 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:36.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.561074 30413 hybrid_clock.cc:648] HybridClock initialized: now 1765746396560988 us; error 78 us; skew 500 ppm
21:06:36.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.562172 30413 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:36.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.570308 30413 webserver.cc:492] Webserver started at http://127.29.139.188:34601/ using document root <none> and password file <none>
21:06:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.571455 30413 fs_manager.cc:362] Metadata directory not provided
21:06:36.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.571732 30413 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:36.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.572283 30413 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:36.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.578495 30413 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/data/instance:
21:06:36.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8261c0f0287b4eb096a12f91830b5267"
21:06:36.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:36 on dist-test-slave-x0xn"
21:06:36.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.580072 30413 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/wal/instance:
21:06:36.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8261c0f0287b4eb096a12f91830b5267"
21:06:36.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:36 on dist-test-slave-x0xn"
21:06:36.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.588943 30413 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.003s sys 0.004s
21:06:36.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.595971 30439 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:36.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.597152 30413 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
21:06:36.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.597522 30413 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/wal
21:06:36.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8261c0f0287b4eb096a12f91830b5267"
21:06:36.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:36 on dist-test-slave-x0xn"
21:06:36.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.597927 30413 fs_report.cc:389] FS layout report
21:06:36.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:36.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/wal
21:06:36.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/wal
21:06:36.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/data/data
21:06:36.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:36.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:36.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.673584 30413 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:36.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.674919 30413 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:36.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.675297 30413 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.700039 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 6
21:06:36.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:36.706732 30335 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.29.139.188:37723: Network error: Client connection negotiation failed: client connection to 127.29.139.188:37723: connect: Connection refused (error 111)
21:06:36.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.742148 30413 rpc_server.cc:307] RPC server started. Bound to: 127.29.139.188:37723
21:06:36.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.742221 30493 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.139.188:37723 every 8 connection(s)
21:06:36.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.745252 30413 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/data/info.pb
21:06:36.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.746891 30254 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 30413
21:06:36.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.747308 30254 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/master-2/wal/instance
21:06:36.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.752610 30494 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:06:36.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.768842 30494 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } has no permanent_uuid. Determining permanent_uuid...
21:06:36.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.792331 30494 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } has no permanent_uuid. Determining permanent_uuid...
21:06:36.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.801672 30494 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } has no permanent_uuid. Determining permanent_uuid...
21:06:36.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.816730 30494 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267: Bootstrap starting.
21:06:36.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.822898 30494 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267: Neither blocks nor log segments found. Creating new log.
21:06:36.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.824816 30494 log.cc:826] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267: Log is configured to *not* fsync() on all Append() calls
21:06:36.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.829406 30494 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267: No bootstrap required, opened a new log
21:06:36.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.846623 30494 raft_consensus.cc:359] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } }
21:06:36.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.847229 30494 raft_consensus.cc:385] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:06:36.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.847486 30494 raft_consensus.cc:740] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8261c0f0287b4eb096a12f91830b5267, State: Initialized, Role: FOLLOWER
21:06:36.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.848547 30494 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } }
21:06:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.851537 30503 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } } }
21:06:36.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.852339 30503 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [sys.catalog]: This master's current role is: FOLLOWER
21:06:36.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.853288 30494 sys_catalog.cc:565] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [sys.catalog]: configured and running, proceeding with master startup.
21:06:36.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:36.875362 30514 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:06:36.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:36.875671 30514 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:06:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.870818 30413 master_runner.cc:186] Error getting master registration for 127.29.139.190:32895: OK, instance_id { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" instance_seqno: 1765746393000269 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:06:36.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:36.890488 30413 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:06:36.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:36.929785 30445 tablet.cc:2378] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:06:37.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.220664 30411 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 7
21:06:37.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.237339 30411 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656: Bootstrap starting.
21:06:37.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.242897 30411 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656: Neither blocks nor log segments found. Creating new log.
21:06:37.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.244652 30411 log.cc:826] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656: Log is configured to *not* fsync() on all Append() calls
21:06:37.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.248703 30411 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656: No bootstrap required, opened a new log
21:06:37.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.254459 30503 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:06:37.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.254958 30503 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } }
21:06:37.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.257319 30503 leader_election.cc:290] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 91288c01a71f43deaf02f5b945a32fe2 (127.29.139.190:32895), a8fef5978fb04a3e83dd90a7a9285656 (127.29.139.189:45383)
21:06:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.273422 30411 raft_consensus.cc:359] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } }
21:06:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.274251 30411 raft_consensus.cc:385] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:06:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.274534 30411 raft_consensus.cc:740] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a8fef5978fb04a3e83dd90a7a9285656, State: Initialized, Role: FOLLOWER
21:06:37.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.275552 30411 consensus_queue.cc:260] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } }
21:06:37.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.278162 30518 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } } }
21:06:37.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.279093 30518 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [sys.catalog]: This master's current role is: FOLLOWER
21:06:37.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.279966 30411 sys_catalog.cc:565] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [sys.catalog]: configured and running, proceeding with master startup.
21:06:37.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.280403 30385 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8261c0f0287b4eb096a12f91830b5267" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8fef5978fb04a3e83dd90a7a9285656" is_pre_election: true
21:06:37.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.281371 30385 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8261c0f0287b4eb096a12f91830b5267 in term 0.
21:06:37.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.282675 30440 leader_election.cc:304] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8261c0f0287b4eb096a12f91830b5267, a8fef5978fb04a3e83dd90a7a9285656; no voters:
21:06:37.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.283578 30503 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [term 0 FOLLOWER]: Leader pre-election won for term 1
21:06:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.283910 30503 raft_consensus.cc:493] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:06:37.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.284262 30503 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [term 0 FOLLOWER]: Advancing to term 1
21:06:37.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:37.288483 30364 tablet.cc:2378] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:06:37.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.294894 30503 raft_consensus.cc:515] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } }
21:06:37.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.298408 30503 leader_election.cc:290] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [CANDIDATE]: Term 1 election: Requested vote from peers 91288c01a71f43deaf02f5b945a32fe2 (127.29.139.190:32895), a8fef5978fb04a3e83dd90a7a9285656 (127.29.139.189:45383)
21:06:37.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.300844 30385 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8261c0f0287b4eb096a12f91830b5267" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a8fef5978fb04a3e83dd90a7a9285656"
21:06:37.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.301378 30385 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [term 0 FOLLOWER]: Advancing to term 1
21:06:37.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.311090 30385 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8261c0f0287b4eb096a12f91830b5267 in term 1.
21:06:37.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.312150 30440 leader_election.cc:304] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8261c0f0287b4eb096a12f91830b5267, a8fef5978fb04a3e83dd90a7a9285656; no voters:
21:06:37.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.312989 30503 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [term 1 FOLLOWER]: Leader election won for term 1
21:06:37.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.316567 30503 raft_consensus.cc:697] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [term 1 LEADER]: Becoming Leader. State: Replica: 8261c0f0287b4eb096a12f91830b5267, State: Running, Role: LEADER
21:06:37.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.317476 30503 consensus_queue.cc:237] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } }
21:06:37.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.327404 30524 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8261c0f0287b4eb096a12f91830b5267. Latest consensus state: current_term: 1 leader_uuid: "8261c0f0287b4eb096a12f91830b5267" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } } }
21:06:37.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.328148 30524 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [sys.catalog]: This master's current role is: LEADER
21:06:37.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.331288 30528 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:06:37.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.346263 30528 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:06:37.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:37.374531 30532 catalog_manager.cc:1568] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:06:37.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:37.376637 30532 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:06:37.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.344378 30337 master_runner.cc:186] Error getting master registration for 127.29.139.190:32895: OK, instance_id { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" instance_seqno: 1765746393000269 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:06:37.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.379837 30337 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:06:37.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.386495 30385 raft_consensus.cc:1275] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [term 1 FOLLOWER]: Refusing update from remote peer 8261c0f0287b4eb096a12f91830b5267: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:06:37.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.387914 30524 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:06:37.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.406877 30518 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8261c0f0287b4eb096a12f91830b5267. Latest consensus state: current_term: 1 leader_uuid: "8261c0f0287b4eb096a12f91830b5267" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } } }
21:06:37.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.407999 30518 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [sys.catalog]: This master's current role is: FOLLOWER
21:06:37.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.411434 30534 mvcc.cc:204] Tried to move back new op lower bound from 7232497243667644416 to 7232497243427373056. Current Snapshot: MvccSnapshot[applied={T|T < 7232497243667644416}]
21:06:37.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.417007 30503 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8261c0f0287b4eb096a12f91830b5267" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } } }
21:06:37.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.417665 30503 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [sys.catalog]: This master's current role is: LEADER
21:06:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.424401 30528 catalog_manager.cc:1357] Generated new cluster ID: b6477eaeb9c1439fa24a69aefc4c5f0b
21:06:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.424669 30528 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:06:37.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.426180 30518 sys_catalog.cc:455] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8261c0f0287b4eb096a12f91830b5267" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } } }
21:06:37.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.426820 30518 sys_catalog.cc:458] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656 [sys.catalog]: This master's current role is: FOLLOWER
21:06:37.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.471676 30528 catalog_manager.cc:1380] Generated new certificate authority record
21:06:37.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.474001 30528 catalog_manager.cc:1514] Loading token signing keys...
21:06:37.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.499192 30528 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267: Generated new TSK 0
21:06:37.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.502390 30528 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:06:37.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:37.550208 30331 debug-util.cc:398] Leaking SignalData structure 0x7b080007fb80 after lost signal to thread 30267
21:06:37.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.558149 30254 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:37.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:37.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/wal
21:06:37.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/data
21:06:37.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/logs
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/data/info.pb
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.129:0
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.139.129
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.129
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:37.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:37.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:06:37.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.743310 30335 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } attempt: 7
21:06:37.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.795349 30335 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2: Bootstrap starting.
21:06:37.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.809755 30335 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2: Neither blocks nor log segments found. Creating new log.
21:06:37.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.822705 30335 log.cc:826] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2: Log is configured to *not* fsync() on all Append() calls
21:06:37.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.852232 30335 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2: No bootstrap required, opened a new log
21:06:37.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.927635 30335 raft_consensus.cc:359] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } }
21:06:37.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.930979 30335 raft_consensus.cc:385] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:06:37.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.931524 30335 raft_consensus.cc:740] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 91288c01a71f43deaf02f5b945a32fe2, State: Initialized, Role: FOLLOWER
21:06:37.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.933282 30335 consensus_queue.cc:260] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } }
21:06:37.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.952699 30335 sys_catalog.cc:565] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [sys.catalog]: configured and running, proceeding with master startup.
21:06:37.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.954356 30307 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [term 0 FOLLOWER]: Advancing to term 1
21:06:37.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.960992 30308 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8261c0f0287b4eb096a12f91830b5267" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "91288c01a71f43deaf02f5b945a32fe2"
21:06:37.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:37.963891 30288 tablet.cc:2378] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:06:37.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.960372 30309 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "8261c0f0287b4eb096a12f91830b5267" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "91288c01a71f43deaf02f5b945a32fe2" is_pre_election: true
21:06:37.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.969398 30307 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [term 1 FOLLOWER]: Refusing update from remote peer 8261c0f0287b4eb096a12f91830b5267: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:06:37.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.971004 30503 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [LEADER]: Connected to new peer: Peer: permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:06:37.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.951401 30543 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } } }
21:06:37.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:37.979809 30543 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [sys.catalog]: This master's current role is: FOLLOWER
21:06:38.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.052738 30543 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 8261c0f0287b4eb096a12f91830b5267. Latest consensus state: current_term: 1 leader_uuid: "8261c0f0287b4eb096a12f91830b5267" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } } }
21:06:38.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.053392 30543 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [sys.catalog]: This master's current role is: FOLLOWER
21:06:38.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.051529 30545 sys_catalog.cc:455] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "8261c0f0287b4eb096a12f91830b5267" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } } }
21:06:38.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.054734 30545 sys_catalog.cc:458] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2 [sys.catalog]: This master's current role is: FOLLOWER
21:06:38.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.101899 30559 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:06:38.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:38.284389 30539 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:38.284974 30539 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:38.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:38.285646 30539 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:38.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:38.345366 30539 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:38.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:38.346704 30539 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.139.129
21:06:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.383021 30532 catalog_manager.cc:1269] Loaded cluster ID: b6477eaeb9c1439fa24a69aefc4c5f0b
21:06:38.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.383626 30532 catalog_manager.cc:1562] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656: loading cluster ID for follower catalog manager: success
21:06:38.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.390790 30532 catalog_manager.cc:1584] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656: acquiring CA information for follower catalog manager: success
21:06:38.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.403740 30532 catalog_manager.cc:1612] T 00000000000000000000000000000000 P a8fef5978fb04a3e83dd90a7a9285656: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:06:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.406435 30539 tablet_server_runner.cc:78] Tablet server non-default flags:
21:06:38.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/data
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/wal
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.129:0
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/data/info.pb
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.129
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.139.129
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/logs
21:06:38.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:06:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:06:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:06:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:06:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:38.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.408102 30539 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:38.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.410157 30539 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:38.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:38.443607 30565 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:38.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.541460 30503 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "8261c0f0287b4eb096a12f91830b5267" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "91288c01a71f43deaf02f5b945a32fe2" member_type: VOTER last_known_addr { host: "127.29.139.190" port: 32895 } } peers { permanent_uuid: "a8fef5978fb04a3e83dd90a7a9285656" member_type: VOTER last_known_addr { host: "127.29.139.189" port: 45383 } } peers { permanent_uuid: "8261c0f0287b4eb096a12f91830b5267" member_type: VOTER last_known_addr { host: "127.29.139.188" port: 37723 } } }
21:06:38.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:38.542497 30503 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8261c0f0287b4eb096a12f91830b5267 [sys.catalog]: This master's current role is: LEADER
21:06:39.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:39.120496 30559 catalog_manager.cc:1269] Loaded cluster ID: b6477eaeb9c1439fa24a69aefc4c5f0b
21:06:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:39.121212 30559 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2: loading cluster ID for follower catalog manager: success
21:06:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:39.129041 30559 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2: acquiring CA information for follower catalog manager: success
21:06:39.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:39.142185 30559 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 91288c01a71f43deaf02f5b945a32fe2: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:06:40.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:39.837597 30564 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 30539
21:06:40.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:38.448588 30566 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:40.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:40.516521 30539 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.070s user 0.408s sys 0.777s
21:06:40.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:40.516922 30539 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.070s user 0.409s sys 0.777s
21:06:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:40.524245 30573 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:40.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.524456 30539 server_base.cc:1047] running on GCE node
21:06:40.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.525800 30539 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:40.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.528111 30539 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:40.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.529584 30539 hybrid_clock.cc:648] HybridClock initialized: now 1765746400529512 us; error 68 us; skew 500 ppm
21:06:40.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.530622 30539 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:40.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.537513 30539 webserver.cc:492] Webserver started at http://127.29.139.129:37033/ using document root <none> and password file <none>
21:06:40.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.538594 30539 fs_manager.cc:362] Metadata directory not provided
21:06:40.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.538856 30539 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:40.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.539391 30539 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:40.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.545616 30539 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/data/instance:
21:06:40.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b315aaff1c754867a7b8848a70adacca"
21:06:40.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:40 on dist-test-slave-x0xn"
21:06:40.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.547194 30539 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/wal/instance:
21:06:40.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b315aaff1c754867a7b8848a70adacca"
21:06:40.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:40 on dist-test-slave-x0xn"
21:06:40.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.556064 30539 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.008s sys 0.000s
21:06:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.563153 30579 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:40.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.564348 30539 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.564708 30539 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/wal
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b315aaff1c754867a7b8848a70adacca"
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:40 on dist-test-slave-x0xn"
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.565112 30539 fs_report.cc:389] FS layout report
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/wal
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/wal
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/data/data
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:40.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:40.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:40.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:40.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:40.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.611496 30539 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:40.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.612843 30539 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:40.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.613209 30539 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:40.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.615847 30539 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:06:40.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.620288 30539 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:06:40.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.620489 30539 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:06:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.620815 30539 ts_tablet_manager.cc:616] Registered 0 tablets
21:06:40.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.621013 30539 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:06:40.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.807169 30539 rpc_server.cc:307] RPC server started. Bound to: 127.29.139.129:35793
21:06:40.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.807340 30692 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.139.129:35793 every 8 connection(s)
21:06:40.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.809669 30539 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/data/info.pb
21:06:40.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.814934 30254 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 30539
21:06:40.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.815410 30254 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-0/wal/instance
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.854614 30254 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/wal
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/data
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/logs
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/data/info.pb
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:40.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.130:0
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.139.130
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.130
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:40.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:06:40.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.869088 30694 heartbeater.cc:344] Connected to a master server at 127.29.139.189:45383
21:06:40.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.869146 30693 heartbeater.cc:344] Connected to a master server at 127.29.139.188:37723
21:06:40.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.869611 30694 heartbeater.cc:461] Registering TS with master...
21:06:40.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.869992 30693 heartbeater.cc:461] Registering TS with master...
21:06:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.870975 30694 heartbeater.cc:507] Master 127.29.139.189:45383 requested a full tablet report, sending...
21:06:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.871032 30693 heartbeater.cc:507] Master 127.29.139.188:37723 requested a full tablet report, sending...
21:06:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.871475 30695 heartbeater.cc:344] Connected to a master server at 127.29.139.190:32895
21:06:40.872 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.871814 30695 heartbeater.cc:461] Registering TS with master...
21:06:40.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.872750 30695 heartbeater.cc:507] Master 127.29.139.190:32895 requested a full tablet report, sending...
21:06:40.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.874972 30456 ts_manager.cc:194] Registered new tserver with Master: b315aaff1c754867a7b8848a70adacca (127.29.139.129:35793)
21:06:40.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.875463 30299 ts_manager.cc:194] Registered new tserver with Master: b315aaff1c754867a7b8848a70adacca (127.29.139.129:35793)
21:06:40.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.875643 30375 ts_manager.cc:194] Registered new tserver with Master: b315aaff1c754867a7b8848a70adacca (127.29.139.129:35793)
21:06:40.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:40.877853 30456 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.139.129:58095
21:06:41.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:41.154024 30704 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:41.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:41.154564 30704 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:41.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:41.155269 30704 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:41.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:41.185829 30704 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:41.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:41.186712 30704 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.139.130
21:06:41.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:41.221391 30704 tablet_server_runner.cc:78] Tablet server non-default flags:
21:06:41.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/data
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/wal
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.130:0
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/data/info.pb
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.130
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.139.130
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/logs
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:06:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:06:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:06:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:06:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:41.222947 30704 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:41.224875 30704 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:41.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:41.239708 30711 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:41.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:41.695907 30407 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac060 after lost signal to thread 30341
21:06:41.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:41.881613 30693 heartbeater.cc:499] Master 127.29.139.188:37723 was elected leader, sending a full tablet report...
21:06:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:42.641558 30710 debug-util.cc:398] Leaking SignalData structure 0x7b080002ec60 after lost signal to thread 30704
21:06:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:42.966920 30710 kernel_stack_watchdog.cc:198] Thread 30704 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
21:06:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:06:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:06:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:06:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:06:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:42.967012 30704 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.727s user 0.557s sys 0.968s
21:06:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:42.967324 30704 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.728s user 0.557s sys 0.968s
21:06:42.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:41.241725 30712 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:42.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:42.968309 30713 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1727 milliseconds
21:06:42.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:42.969985 30714 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:42.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:42.970594 30704 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:06:42.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:42.973526 30704 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:42.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:42.976075 30704 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:42.977 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:42.977505 30704 hybrid_clock.cc:648] HybridClock initialized: now 1765746402977400 us; error 98 us; skew 500 ppm
21:06:42.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:42.978508 30704 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:42.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:42.985508 30704 webserver.cc:492] Webserver started at http://127.29.139.130:35455/ using document root <none> and password file <none>
21:06:42.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:42.986632 30704 fs_manager.cc:362] Metadata directory not provided
21:06:42.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:42.986892 30704 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:42.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:42.987381 30704 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:42.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:42.993275 30704 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/data/instance:
21:06:42.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7ec714fd0c1e4fb5b361c7484e54ee65"
21:06:42.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:42 on dist-test-slave-x0xn"
21:06:42.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:42.994685 30704 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/wal/instance:
21:06:42.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7ec714fd0c1e4fb5b361c7484e54ee65"
21:06:42.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:42 on dist-test-slave-x0xn"
21:06:43.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.003115 30704 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.006s sys 0.004s
21:06:43.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.010126 30721 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:43.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.011372 30704 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
21:06:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.011770 30704 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/wal
21:06:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "7ec714fd0c1e4fb5b361c7484e54ee65"
21:06:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:42 on dist-test-slave-x0xn"
21:06:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.012228 30704 fs_report.cc:389] FS layout report
21:06:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/wal
21:06:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/wal
21:06:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/data/data
21:06:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:43.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:43.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:43.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.080029 30704 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.081965 30704 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:43.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.082499 30704 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:43.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.085681 30704 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:06:43.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.091470 30704 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:06:43.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.091758 30704 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:06:43.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.092041 30704 ts_tablet_manager.cc:616] Registered 0 tablets
21:06:43.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.092274 30704 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
21:06:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.241222 30704 rpc_server.cc:307] RPC server started. Bound to: 127.29.139.130:41971
21:06:43.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.241291 30834 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.139.130:41971 every 8 connection(s)
21:06:43.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.244630 30704 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/data/info.pb
21:06:43.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.254307 30254 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 30704
21:06:43.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.254760 30254 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-1/wal/instance
21:06:43.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.274276 30254 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/wal
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/data
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/logs
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/data/info.pb
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.131:0
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.139.131
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.131
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:43.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:43.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:43.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:43.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
21:06:43.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.316367 30837 heartbeater.cc:344] Connected to a master server at 127.29.139.190:32895
21:06:43.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.316887 30837 heartbeater.cc:461] Registering TS with master...
21:06:43.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.318203 30837 heartbeater.cc:507] Master 127.29.139.190:32895 requested a full tablet report, sending...
21:06:43.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.319137 30835 heartbeater.cc:344] Connected to a master server at 127.29.139.188:37723
21:06:43.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.319455 30835 heartbeater.cc:461] Registering TS with master...
21:06:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.320922 30836 heartbeater.cc:344] Connected to a master server at 127.29.139.189:45383
21:06:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.321244 30836 heartbeater.cc:461] Registering TS with master...
21:06:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.321211 30835 heartbeater.cc:507] Master 127.29.139.188:37723 requested a full tablet report, sending...
21:06:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.321739 30299 ts_manager.cc:194] Registered new tserver with Master: 7ec714fd0c1e4fb5b361c7484e54ee65 (127.29.139.130:41971)
21:06:43.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.322106 30836 heartbeater.cc:507] Master 127.29.139.189:45383 requested a full tablet report, sending...
21:06:43.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.324457 30375 ts_manager.cc:194] Registered new tserver with Master: 7ec714fd0c1e4fb5b361c7484e54ee65 (127.29.139.130:41971)
21:06:43.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.324519 30456 ts_manager.cc:194] Registered new tserver with Master: 7ec714fd0c1e4fb5b361c7484e54ee65 (127.29.139.130:41971)
21:06:43.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.326107 30456 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.139.130:37559
21:06:43.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:43.602255 30842 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:43.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:43.602713 30842 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:43.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:43.603149 30842 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:43.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:43.634232 30842 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:43.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:43.635003 30842 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.29.139.131
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.668744 30842 tablet_server_runner.cc:78] Tablet server non-default flags:
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.29.139.148:40635
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/data
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/wal
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.29.139.131:0
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:43.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/data/info.pb
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.29.139.131
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.29.139.190:32895,127.29.139.189:45383,127.29.139.188:37723
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.29.139.131
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/logs
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:43.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.669930 30842 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:43.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:43.671626 30842 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:43.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:43.683626 30852 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:44.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:44.329650 30835 heartbeater.cc:499] Master 127.29.139.188:37723 was elected leader, sending a full tablet report...
21:06:45.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:43.684553 30853 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:45.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:45.012687 30854 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1324 milliseconds
21:06:45.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:45.014374 30855 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:45.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:45.016489 30842 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.331s user 0.007s sys 0.002s
21:06:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:45.016816 30842 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.332s user 0.007s sys 0.002s
21:06:45.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.017023 30842 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:06:45.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.018201 30842 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:45.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.020695 30842 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:45.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.022078 30842 hybrid_clock.cc:648] HybridClock initialized: now 1765746405022021 us; error 61 us; skew 500 ppm
21:06:45.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.022830 30842 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:45.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.030123 30842 webserver.cc:492] Webserver started at http://127.29.139.131:35745/ using document root <none> and password file <none>
21:06:45.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.030966 30842 fs_manager.cc:362] Metadata directory not provided
21:06:45.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.031145 30842 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:45.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.031515 30842 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:45.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.035547 30842 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/data/instance:
21:06:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fef95b611a5647ec89cc1ca151fe3531"
21:06:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:45 on dist-test-slave-x0xn"
21:06:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.036583 30842 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/wal/instance:
21:06:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fef95b611a5647ec89cc1ca151fe3531"
21:06:45.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:45 on dist-test-slave-x0xn"
21:06:45.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.044040 30842 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
21:06:45.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.049666 30862 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:45.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.050676 30842 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
21:06:45.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.050935 30842 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/wal
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "fef95b611a5647ec89cc1ca151fe3531"
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:45 on dist-test-slave-x0xn"
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.051198 30842 fs_report.cc:389] FS layout report
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/wal
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/wal
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/data/data
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:45.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:45.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.118268 30842 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:45.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.119604 30842 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:45.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.119997 30842 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:45.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.123170 30842 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:06:45.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.127112 30842 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:06:45.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.127312 30842 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:06:45.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.127658 30842 ts_tablet_manager.cc:616] Registered 0 tablets
21:06:45.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.127825 30842 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:06:45.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.259694 30842 rpc_server.cc:307] RPC server started. Bound to: 127.29.139.131:43945
21:06:45.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.259804 30976 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.29.139.131:43945 every 8 connection(s)
21:06:45.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.262244 30842 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/data/info.pb
21:06:45.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.272684 30254 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 30842
21:06:45.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.273144 30254 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster3117719538186271811/ts-2/wal/instance
21:06:45.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.310292 30978 heartbeater.cc:344] Connected to a master server at 127.29.139.189:45383
21:06:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.310806 30978 heartbeater.cc:461] Registering TS with master...
21:06:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.310842 30979 heartbeater.cc:344] Connected to a master server at 127.29.139.190:32895
21:06:45.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.311393 30979 heartbeater.cc:461] Registering TS with master...
21:06:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.311957 30977 heartbeater.cc:344] Connected to a master server at 127.29.139.188:37723
21:06:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.312321 30978 heartbeater.cc:507] Master 127.29.139.189:45383 requested a full tablet report, sending...
21:06:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.312500 30977 heartbeater.cc:461] Registering TS with master...
21:06:45.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.312484 30979 heartbeater.cc:507] Master 127.29.139.190:32895 requested a full tablet report, sending...
21:06:45.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.313463 30977 heartbeater.cc:507] Master 127.29.139.188:37723 requested a full tablet report, sending...
21:06:45.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.315167 30298 ts_manager.cc:194] Registered new tserver with Master: fef95b611a5647ec89cc1ca151fe3531 (127.29.139.131:43945)
21:06:45.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.315151 30375 ts_manager.cc:194] Registered new tserver with Master: fef95b611a5647ec89cc1ca151fe3531 (127.29.139.131:43945)
21:06:45.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.316076 30456 ts_manager.cc:194] Registered new tserver with Master: fef95b611a5647ec89cc1ca151fe3531 (127.29.139.131:43945)
21:06:45.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.317587 30456 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.29.139.131:49367
21:06:45.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:45.329030 30254 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:06:45.341 [DEBUG - main] (MiniKuduCluster.java:195) Response:
21:06:45.343 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
21:06:45.357 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.29.139.190"
port: 32895
}
bound_http_address {
host: "127.29.139.190"
port: 40747
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.29.139.189"
port: 45383
}
bound_http_address {
host: "127.29.139.189"
port: 37393
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.29.139.188"
port: 37723
}
bound_http_address {
host: "127.29.139.188"
port: 34601
}
}
}
21:06:45.362 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:06:45.365 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.29.139.129"
port: 35793
}
bound_http_address {
host: "127.29.139.129"
port: 37033
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.29.139.130"
port: 41971
}
bound_http_address {
host: "127.29.139.130"
port: 35455
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.29.139.131"
port: 43945
}
bound_http_address {
host: "127.29.139.131"
port: 35745
}
}
}
21:06:45.365 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
21:06:45.553 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
21:06:45.577 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
21:06:45.691 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskNTuEsu/test-tmp
21:06:45.691 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
21:06:45.692 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskNTuEsu/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
21:06:45.694 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
21:06:46.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:06:46.002 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
21:06:46.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:46.005123 30988 mini_oidc.cc:150] Starting JWKS server
21:06:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:46.008268 30988 webserver.cc:492] Webserver started at http://127.0.0.1:37431/ using document root <none> and password file <none>
21:06:46.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:46.008736 30988 mini_oidc.cc:186] Starting OIDC Discovery server
21:06:46.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:46.010362 30988 webserver.cc:492] Webserver started at http://127.0.0.1:36727/ using document root <none> and password file <none>
21:06:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
21:06:46.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/principal' for realm 'KRBTEST.COM',
21:06:46.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
21:06:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:46 dist-test-slave-x0xn krb5kdc[31000](info): setting up network...
21:06:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
21:06:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:46 dist-test-slave-x0xn krb5kdc[31000](info): set up 2 sockets
21:06:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:46 dist-test-slave-x0xn krb5kdc[31000](info): commencing operation
21:06:46.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
21:06:46.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:46.320336 30977 heartbeater.cc:499] Master 127.29.139.188:37723 was elected leader, sending a full tablet report...
21:06:47.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:47.914114 30988 mini_kdc.cc:121] Time spent starting KDC: real 1.872s user 0.008s sys 0.008s
21:06:47.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
21:06:47.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:06:47.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
21:06:47.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
21:06:47.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:06:47.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
21:06:47.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
21:06:47.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:06:47.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
21:06:48.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
21:06:48.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/test-user.keytab.
21:06:48.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/test-user.keytab.
21:06:48.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:48 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746408, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:06:48.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:06:48.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-14T21:06:48Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
21:06:48.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-14T21:06:48Z Disabled control of system clock
21:06:48.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.67.62@KRBTEST.COM; defaulting to no policy
21:06:48.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:48.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.67.62@KRBTEST.COM" created.
21:06:48.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:48.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:48.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.30.67.62@KRBTEST.COM; defaulting to no policy
21:06:48.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:48.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.30.67.62@KRBTEST.COM" created.
21:06:48.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:48.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.62 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:48.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.62 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:48.097973 30988 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/logs
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data/info.pb
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:48.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.62:45687
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.62
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.62
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:06:48.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:06:48.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:06:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.391878 31017 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:48.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.392402 31017 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:48.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.393002 31017 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:48.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.423022 31017 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:06:48.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.423228 31017 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:06:48.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.423484 31017 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:48.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.423684 31017 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:06:48.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.423869 31017 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:06:48.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.424052 31017 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:06:48.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.424213 31017 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37431/jwks.json
21:06:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:48.459753 31017 master_runner.cc:386] Master server non-default flags:
21:06:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:06:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:48.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.62
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.62:45687
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data/info.pb
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.62
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/logs
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:48.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:48.460954 31017 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:48.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:48.461946 31017 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:48.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.472424 31024 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:49.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:48.474408 31025 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:49.713644 31027 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:49.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:49.715723 31017 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.243s user 0.497s sys 0.739s
21:06:49.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:49.716272 31017 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.244s user 0.499s sys 0.741s
21:06:49.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:49.716543 31026 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1238 milliseconds
21:06:49.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.716576 31017 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:06:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.717793 31017 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:49.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.720994 31017 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.722527 31017 hybrid_clock.cc:648] HybridClock initialized: now 1765746409722437 us; error 86 us; skew 500 ppm
21:06:49.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.723316 31017 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:49.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:49 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746409, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:06:49.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.729795 31017 init.cc:377] Logged in from keytab as kudu/127.30.67.62@KRBTEST.COM (short username kudu)
21:06:49.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.734750 31017 webserver.cc:492] Webserver started at http://127.30.67.62:43597/ using document root <none> and password file <none>
21:06:49.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.735560 31017 fs_manager.cc:362] Metadata directory not provided
21:06:49.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.735741 31017 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:49.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.736151 31017 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:49.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.740352 31017 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data/instance:
21:06:49.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "422367ec3d01457bb1fb5b6a151edb46"
21:06:49.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:49 on dist-test-slave-x0xn"
21:06:49.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.742027 31017 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal/instance:
21:06:49.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "422367ec3d01457bb1fb5b6a151edb46"
21:06:49.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:49 on dist-test-slave-x0xn"
21:06:49.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.751416 31017 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.005s sys 0.005s
21:06:49.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.763034 31035 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:49.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.764425 31017 fs_manager.cc:730] Time spent opening block manager: real 0.009s user 0.011s sys 0.000s
21:06:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.764815 31017 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal
21:06:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "422367ec3d01457bb1fb5b6a151edb46"
21:06:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:49 on dist-test-slave-x0xn"
21:06:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.765213 31017 fs_report.cc:389] FS layout report
21:06:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal
21:06:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal
21:06:49.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data/data
21:06:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:49.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:49.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.852958 31017 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:49.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.858336 31017 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:49.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.858734 31017 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:49.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.925201 31017 rpc_server.cc:307] RPC server started. Bound to: 127.30.67.62:45687
21:06:49.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.925269 31086 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.67.62:45687 every 8 connection(s)
21:06:49.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.928581 31017 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data/info.pb
21:06:49.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.935073 31087 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:06:49.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.935341 30988 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 31017
21:06:49.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.935808 30988 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal/instance
21:06:49.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.67.61@KRBTEST.COM; defaulting to no policy
21:06:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.67.61@KRBTEST.COM" created.
21:06:49.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.949599 31087 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } has no permanent_uuid. Determining permanent_uuid...
21:06:49.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:49 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746409, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.62@KRBTEST.COM for kudu/127.30.67.62@KRBTEST.COM
21:06:49.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:49.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:49.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:49.985713 31087 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } has no permanent_uuid. Determining permanent_uuid...
21:06:49.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.30.67.61@KRBTEST.COM; defaulting to no policy
21:06:49.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:49.988014 31038 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.67.61:41887: connect: Connection refused (error 111)
21:06:49.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:49.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.30.67.61@KRBTEST.COM" created.
21:06:49.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:49.992031 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.61:41887: Network error: Client connection negotiation failed: client connection to 127.30.67.61:41887: connect: Connection refused (error 111)
21:06:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.61 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:50.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.61 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:50.005782 30988 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/logs
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data/info.pb
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.61:41887
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.61
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:50.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.61
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:06:50.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:06:50.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:50.041504 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } attempt: 1
21:06:50.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.045230 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.61:41887: Network error: Client connection negotiation failed: client connection to 127.30.67.61:41887: connect: Connection refused (error 111)
21:06:50.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:50.113583 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } attempt: 2
21:06:50.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.117204 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.61:41887: Network error: Client connection negotiation failed: client connection to 127.30.67.61:41887: connect: Connection refused (error 111)
21:06:50.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:50.208539 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } attempt: 3
21:06:50.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.212018 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.61:41887: Network error: Client connection negotiation failed: client connection to 127.30.67.61:41887: connect: Connection refused (error 111)
21:06:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.304207 31096 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:50.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.304687 31096 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:50.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.305279 31096 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:50.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.337320 31096 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:06:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.337860 31096 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:06:50.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.338457 31096 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.338932 31096 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:06:50.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.339361 31096 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:06:50.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.339813 31096 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:06:50.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.340193 31096 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37431/jwks.json
21:06:50.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:50.355406 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } attempt: 4
21:06:50.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.358964 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.61:41887: Network error: Client connection negotiation failed: client connection to 127.30.67.61:41887: connect: Connection refused (error 111)
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:50.379947 31096 master_runner.cc:386] Master server non-default flags:
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.61
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.61:41887
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data/info.pb
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.61
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:50.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/logs
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:50.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:50.381125 31096 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:50.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:50.382017 31096 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:50.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.392010 31102 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:50.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:50.658290 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } attempt: 5
21:06:50.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.663722 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.61:41887: Network error: Client connection negotiation failed: client connection to 127.30.67.61:41887: connect: Connection refused (error 111)
21:06:51.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.211150 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } attempt: 6
21:06:51.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:51.236167 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.61:41887: Network error: Client connection negotiation failed: client connection to 127.30.67.61:41887: connect: Connection refused (error 111)
21:06:51.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:51.793978 31101 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 31096
21:06:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:51.868402 31101 kernel_stack_watchdog.cc:198] Thread 31096 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
21:06:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:06:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:06:51.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:06:51.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:06:51.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:51.868770 31096 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.477s user 0.579s sys 0.843s
21:06:51.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:51.869107 31096 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.477s user 0.579s sys 0.843s
21:06:51.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:50.392613 31103 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:51.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:51.870864 31104 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1478 milliseconds
21:06:51.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:51.870819 31105 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:51.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.870792 31096 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:06:51.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.874437 31096 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.876734 31096 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.878083 31096 hybrid_clock.cc:648] HybridClock initialized: now 1765746411878040 us; error 39 us; skew 500 ppm
21:06:51.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.878911 31096 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:51.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:51 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746411, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:06:51.884 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.884377 31096 init.cc:377] Logged in from keytab as kudu/127.30.67.61@KRBTEST.COM (short username kudu)
21:06:51.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.888607 31096 webserver.cc:492] Webserver started at http://127.30.67.61:40011/ using document root <none> and password file <none>
21:06:51.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.889456 31096 fs_manager.cc:362] Metadata directory not provided
21:06:51.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.889667 31096 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:51.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.890125 31096 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.894136 31096 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data/instance:
21:06:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73433c1d8df9487bb1424ca34764d4db"
21:06:51.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:51 on dist-test-slave-x0xn"
21:06:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.895328 31096 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal/instance:
21:06:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73433c1d8df9487bb1424ca34764d4db"
21:06:51.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:51 on dist-test-slave-x0xn"
21:06:51.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.902196 31096 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.004s sys 0.004s
21:06:51.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.907106 31118 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:51.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.908027 31096 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
21:06:51.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.908365 31096 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal
21:06:51.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73433c1d8df9487bb1424ca34764d4db"
21:06:51.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:51 on dist-test-slave-x0xn"
21:06:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.908665 31096 fs_report.cc:389] FS layout report
21:06:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal
21:06:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal
21:06:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data/data
21:06:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:51.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:51.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:51.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.989665 31096 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:51.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.997821 31096 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:51.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:51.998208 31096 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.065500 31096 rpc_server.cc:307] RPC server started. Bound to: 127.30.67.61:41887
21:06:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.065584 31169 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.67.61:41887 every 8 connection(s)
21:06:52.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.067971 31096 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data/info.pb
21:06:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.073076 30988 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 31096
21:06:52.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.073393 30988 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal/instance
21:06:52.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.074426 31170 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:06:52.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.67.60@KRBTEST.COM; defaulting to no policy
21:06:52.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.084450 31170 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } has no permanent_uuid. Determining permanent_uuid...
21:06:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:52.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.67.60@KRBTEST.COM" created.
21:06:52.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:52 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746411, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.61@KRBTEST.COM for kudu/127.30.67.62@KRBTEST.COM
21:06:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:52.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:52.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.115386 31170 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } has no permanent_uuid. Determining permanent_uuid...
21:06:52.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.30.67.60@KRBTEST.COM; defaulting to no policy
21:06:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:52.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.30.67.60@KRBTEST.COM" created.
21:06:52.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:52 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746411, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.61@KRBTEST.COM for kudu/127.30.67.61@KRBTEST.COM
21:06:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.60 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.60 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:52.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.139273 31170 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } has no permanent_uuid. Determining permanent_uuid...
21:06:52.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.141482 31119 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.141813 30988 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/logs
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data/info.pb
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:06:52.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.60:41497
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.60
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.60
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:06:52.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:06:52.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.146287 31170 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.195703 31170 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 1
21:06:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.199155 31170 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.267524 31170 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 2
21:06:52.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.272483 31170 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.296584 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } attempt: 7
21:06:52.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:52 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746409, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.62@KRBTEST.COM for kudu/127.30.67.61@KRBTEST.COM
21:06:52.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.316256 31087 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } has no permanent_uuid. Determining permanent_uuid...
21:06:52.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.321681 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.364044 31170 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 3
21:06:52.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.368075 31170 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.380028 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 1
21:06:52.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.383689 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.445556 31180 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.446089 31180 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.446754 31180 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:52.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.465029 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 2
21:06:52.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.468698 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.478639 31180 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:06:52.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.478838 31180 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:06:52.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.479103 31180 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:52.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.479326 31180 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:06:52.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.479524 31180 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:06:52.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.479720 31180 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:06:52.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.479866 31180 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37431/jwks.json
21:06:52.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.511466 31170 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 4
21:06:52.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.514891 31170 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.515759 31180 master_runner.cc:386] Master server non-default flags:
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.60
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.60:41497
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data/info.pb
21:06:52.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.60
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/logs
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:52.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.517237 31180 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:52.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.518476 31180 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:52.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.528779 31188 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:52.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.554688 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 3
21:06:52.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.561128 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.701576 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 4
21:06:52.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.707227 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.814216 31170 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 5
21:06:52.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.820655 31170 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:52.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:52.991348 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 5
21:06:53.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:53.000063 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:53.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:53.369921 31170 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 6
21:06:53.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:53.406492 31170 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:53.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:53.552507 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 6
21:06:53.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:53.571626 31087 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:06:54.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:54.040302 30689 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad060 after lost signal to thread 30560
21:06:54.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:54.041450 30689 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad1c0 after lost signal to thread 30692
21:06:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:52.529807 31189 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:53.931021 31187 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 31180
21:06:54.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:54.160786 31180 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.631s user 0.644s sys 0.982s
21:06:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:54.163337 31191 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:54.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:54.162876 31190 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1632 milliseconds
21:06:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:54.163880 31180 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.634s user 0.644s sys 0.983s
21:06:54.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.164147 31180 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:06:54.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.167474 31180 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:54.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.170324 31180 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:54.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.171871 31180 hybrid_clock.cc:648] HybridClock initialized: now 1765746414171818 us; error 45 us; skew 500 ppm
21:06:54.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.172916 31180 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:54.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:54 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746414, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:06:54.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.179863 31180 init.cc:377] Logged in from keytab as kudu/127.30.67.60@KRBTEST.COM (short username kudu)
21:06:54.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.185142 31180 webserver.cc:492] Webserver started at http://127.30.67.60:41617/ using document root <none> and password file <none>
21:06:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.186270 31180 fs_manager.cc:362] Metadata directory not provided
21:06:54.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.186514 31180 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:54.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.187031 31180 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:54.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.192937 31180 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data/instance:
21:06:54.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65976ecfedef4b369da0bd68c0ff8356"
21:06:54.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:54 on dist-test-slave-x0xn"
21:06:54.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.194342 31180 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal/instance:
21:06:54.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65976ecfedef4b369da0bd68c0ff8356"
21:06:54.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:54 on dist-test-slave-x0xn"
21:06:54.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.203329 31180 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.007s sys 0.000s
21:06:54.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.209915 31205 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.211032 31180 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.002s
21:06:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.211385 31180 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal
21:06:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65976ecfedef4b369da0bd68c0ff8356"
21:06:54.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:54 on dist-test-slave-x0xn"
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.211984 31180 fs_report.cc:389] FS layout report
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data/data
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:54.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:54.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.278304 31180 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:54.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.284011 31180 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:54.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.284493 31180 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.370666 31180 rpc_server.cc:307] RPC server started. Bound to: 127.30.67.60:41497
21:06:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.370731 31256 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.67.60:41497 every 8 connection(s)
21:06:54.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.373279 31180 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data/info.pb
21:06:54.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.374454 30988 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 31180
21:06:54.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.374949 30988 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal/instance
21:06:54.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.384485 31257 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
21:06:54.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.400745 31257 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } has no permanent_uuid. Determining permanent_uuid...
21:06:54.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:54 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746414, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.60@KRBTEST.COM for kudu/127.30.67.62@KRBTEST.COM
21:06:54.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:54 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746408, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.67.62@KRBTEST.COM
21:06:54.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.461982 31257 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } has no permanent_uuid. Determining permanent_uuid...
21:06:54.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.466945 31170 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 7
21:06:54.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:54 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746414, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.60@KRBTEST.COM for kudu/127.30.67.61@KRBTEST.COM
21:06:54.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:54 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746408, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.67.61@KRBTEST.COM
21:06:54.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.491262 31257 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } has no permanent_uuid. Determining permanent_uuid...
21:06:54.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:54 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746411, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.61@KRBTEST.COM for kudu/127.30.67.60@KRBTEST.COM
21:06:54.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:54 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746408, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.30.67.60@KRBTEST.COM
21:06:54.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:54 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746414, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.60@KRBTEST.COM for kudu/127.30.67.60@KRBTEST.COM
21:06:54.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.523948 31170 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: Bootstrap starting.
21:06:54.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.530952 31170 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: Neither blocks nor log segments found. Creating new log.
21:06:54.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.532992 31170 log.cc:826] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: Log is configured to *not* fsync() on all Append() calls
21:06:54.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.540985 31170 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: No bootstrap required, opened a new log
21:06:54.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.544193 31257 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: Bootstrap starting.
21:06:54.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.551198 31257 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: Neither blocks nor log segments found. Creating new log.
21:06:54.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.553020 31257 log.cc:826] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: Log is configured to *not* fsync() on all Append() calls
21:06:54.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.563072 31257 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: No bootstrap required, opened a new log
21:06:54.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.566561 31170 raft_consensus.cc:359] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:06:54.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.567353 31170 raft_consensus.cc:385] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:06:54.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.567657 31170 raft_consensus.cc:740] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73433c1d8df9487bb1424ca34764d4db, State: Initialized, Role: FOLLOWER
21:06:54.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.568522 31170 consensus_queue.cc:260] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:06:54.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.577101 31271 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:06:54.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.578012 31271 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.578857 31170 sys_catalog.cc:565] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: configured and running, proceeding with master startup.
21:06:54.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.584422 31257 raft_consensus.cc:359] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:06:54.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.585223 31257 raft_consensus.cc:385] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:06:54.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.585502 31257 raft_consensus.cc:740] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 65976ecfedef4b369da0bd68c0ff8356, State: Initialized, Role: FOLLOWER
21:06:54.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.586465 31257 consensus_queue.cc:260] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:06:54.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.590097 31274 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:06:54.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.590965 31274 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.592083 31257 sys_catalog.cc:565] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: configured and running, proceeding with master startup.
21:06:54.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.605067 31087 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } attempt: 7
21:06:54.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:54 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746409, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.62@KRBTEST.COM for kudu/127.30.67.60@KRBTEST.COM
21:06:54.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.649081 31087 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: Bootstrap starting.
21:06:54.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.658550 31087 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: Neither blocks nor log segments found. Creating new log.
21:06:54.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.662833 31087 log.cc:826] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: Log is configured to *not* fsync() on all Append() calls
21:06:54.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.670442 31292 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:06:54.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.676476 31087 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: No bootstrap required, opened a new log
21:06:54.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.696444 31296 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:06:54.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.702863 31087 raft_consensus.cc:359] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:06:54.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.703961 31087 raft_consensus.cc:385] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
21:06:54.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.704430 31087 raft_consensus.cc:740] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 422367ec3d01457bb1fb5b6a151edb46, State: Initialized, Role: FOLLOWER
21:06:54.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.705386 31087 consensus_queue.cc:260] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:06:54.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.709904 31271 raft_consensus.cc:493] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:06:54.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.710367 31271 raft_consensus.cc:515] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:06:54.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.712726 31271 leader_election.cc:290] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 422367ec3d01457bb1fb5b6a151edb46 (127.30.67.62:45687), 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497)
21:06:54.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.714651 31232 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "73433c1d8df9487bb1424ca34764d4db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "65976ecfedef4b369da0bd68c0ff8356" is_pre_election: true
21:06:54.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.715370 31232 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 73433c1d8df9487bb1424ca34764d4db in term 0.
21:06:54.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.718086 31119 leader_election.cc:304] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 65976ecfedef4b369da0bd68c0ff8356, 73433c1d8df9487bb1424ca34764d4db; no voters:
21:06:54.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.720062 31271 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 0 FOLLOWER]: Leader pre-election won for term 1
21:06:54.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.673918 31096 master_runner.cc:186] Error getting master registration for 127.30.67.62:45687: OK, instance_id { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" instance_seqno: 1765746409905412 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:06:54.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.720405 31271 raft_consensus.cc:493] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:06:54.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.720785 31271 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 0 FOLLOWER]: Advancing to term 1
21:06:54.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.725693 31297 sys_catalog.cc:455] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:06:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.727731 31087 sys_catalog.cc:565] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: configured and running, proceeding with master startup.
21:06:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.727416 31271 raft_consensus.cc:515] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:06:54.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.728338 31096 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:06:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.728497 31062 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "73433c1d8df9487bb1424ca34764d4db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "422367ec3d01457bb1fb5b6a151edb46" is_pre_election: true
21:06:54.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.729275 31062 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 73433c1d8df9487bb1424ca34764d4db in term 0.
21:06:54.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.736608 31062 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "73433c1d8df9487bb1424ca34764d4db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "422367ec3d01457bb1fb5b6a151edb46"
21:06:54.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.737078 31062 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 0 FOLLOWER]: Advancing to term 1
21:06:54.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.737686 31232 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "73433c1d8df9487bb1424ca34764d4db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "65976ecfedef4b369da0bd68c0ff8356"
21:06:54.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.738202 31232 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [term 0 FOLLOWER]: Advancing to term 1
21:06:54.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.743697 31062 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 73433c1d8df9487bb1424ca34764d4db in term 1.
21:06:54.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.745218 31120 leader_election.cc:304] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 422367ec3d01457bb1fb5b6a151edb46, 73433c1d8df9487bb1424ca34764d4db; no voters:
21:06:54.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.744204 31297 sys_catalog.cc:458] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:54.754980 31124 tablet.cc:2378] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:06:54.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.757283 31232 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 73433c1d8df9487bb1424ca34764d4db in term 1.
21:06:54.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.758685 31271 leader_election.cc:290] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 1 election: Requested vote from peers 422367ec3d01457bb1fb5b6a151edb46 (127.30.67.62:45687), 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497)
21:06:54.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.760326 31299 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Leader election won for term 1
21:06:54.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.769573 31299 raft_consensus.cc:697] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 LEADER]: Becoming Leader. State: Replica: 73433c1d8df9487bb1424ca34764d4db, State: Running, Role: LEADER
21:06:54.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.770569 31299 consensus_queue.cc:237] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:06:54.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.700210 31180 master_runner.cc:186] Error getting master registration for 127.30.67.62:45687: OK, instance_id { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" instance_seqno: 1765746409905412 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
21:06:54.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.783725 31180 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:06:54.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.787369 31271 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: SysCatalogTable state changed. Reason: New leader 73433c1d8df9487bb1424ca34764d4db. Latest consensus state: current_term: 1 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:06:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:54.790227 31211 tablet.cc:2378] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:06:54.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.791299 31271 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: This master's current role is: LEADER
21:06:54.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.798861 31304 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:06:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.817914 31304 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:06:54.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.846167 31062 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 1 FOLLOWER]: Refusing update from remote peer 73433c1d8df9487bb1424ca34764d4db: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:06:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.847978 31299 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [LEADER]: Connected to new peer: Peer: permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:06:54.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.848788 31232 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [term 1 FOLLOWER]: Refusing update from remote peer 73433c1d8df9487bb1424ca34764d4db: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
21:06:54.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.850381 31299 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [LEADER]: Connected to new peer: Peer: permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
21:06:54.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:54.856032 31313 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
21:06:54.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:54.856344 31313 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
21:06:54.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:54.868984 31041 tablet.cc:2378] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: Can't schedule compaction. Clean time has not been advanced past its initial value.
21:06:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.899223 31274 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 73433c1d8df9487bb1424ca34764d4db. Latest consensus state: current_term: 1 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:06:54.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.899936 31274 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.910331 31314 mvcc.cc:204] Tried to move back new op lower bound from 7232497315190489088 to 7232497314913411072. Current Snapshot: MvccSnapshot[applied={T|T < 7232497315190489088}]
21:06:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.916383 31297 sys_catalog.cc:455] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 73433c1d8df9487bb1424ca34764d4db. Latest consensus state: current_term: 1 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:06:54.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.917093 31297 sys_catalog.cc:458] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.923081 31316 mvcc.cc:204] Tried to move back new op lower bound from 7232497315190489088 to 7232497314913411072. Current Snapshot: MvccSnapshot[applied={T|T < 7232497315190489088}]
21:06:54.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.933542 31274 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:06:54.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.934178 31274 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.937369 31297 sys_catalog.cc:455] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:06:54.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.937996 31297 sys_catalog.cc:458] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: This master's current role is: FOLLOWER
21:06:54.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.940214 31300 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:06:54.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.940912 31300 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: This master's current role is: LEADER
21:06:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.941903 31300 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:06:54.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.942498 31300 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: This master's current role is: LEADER
21:06:54.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.945297 31304 catalog_manager.cc:1357] Generated new cluster ID: a48b2d3c23b5424ea5338a0d99b73d85
21:06:54.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.945598 31304 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:06:54.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.987102 31304 catalog_manager.cc:1380] Generated new certificate authority record
21:06:54.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:54.989050 31304 catalog_manager.cc:1514] Loading token signing keys...
21:06:55.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:55.043725 31304 catalog_manager.cc:6027] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: Generated new TSK 0
21:06:55.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:55.044801 31304 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:06:55.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.67.1@KRBTEST.COM; defaulting to no policy
21:06:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:55.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.67.1@KRBTEST.COM" created.
21:06:55.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:55.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:55.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:55.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.30.67.1@KRBTEST.COM; defaulting to no policy
21:06:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:55.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.30.67.1@KRBTEST.COM" created.
21:06:55.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:55.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.1 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:55.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.1 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:55.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:55.198697 30988 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:55.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:55.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/wal
21:06:55.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/data
21:06:55.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/logs
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/data/info.pb
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.1:0
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.67.1
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.1
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.1
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:55.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:06:55.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:55.724926 31296 catalog_manager.cc:1269] Loaded cluster ID: a48b2d3c23b5424ea5338a0d99b73d85
21:06:55.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:55.725474 31296 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: loading cluster ID for follower catalog manager: success
21:06:55.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:55.744647 31296 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: acquiring CA information for follower catalog manager: success
21:06:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:55.749770 31296 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:06:55.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:55.861553 31313 catalog_manager.cc:1269] Loaded cluster ID: a48b2d3c23b5424ea5338a0d99b73d85
21:06:55.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:55.862075 31313 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: loading cluster ID for follower catalog manager: success
21:06:55.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:55.875182 31313 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: acquiring CA information for follower catalog manager: success
21:06:55.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:55.883343 31313 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:06:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:55.903992 31329 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:55.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:55.904629 31329 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:55.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:55.905622 31329 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:55.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:55.973721 31329 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:55.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:55.975281 31329 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.67.1
21:06:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:56.049973 31329 tablet_server_runner.cc:78] Tablet server non-default flags:
21:06:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:06:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/data
21:06:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/wal
21:06:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:06:56.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.1
21:06:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.1:0
21:06:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:06:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:06:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:06:56.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/data/info.pb
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.1
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.67.1
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/logs
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:56.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:56.052320 31329 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:56.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:56.053859 31329 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:56.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:56.082744 31338 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:58.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:57.476395 31337 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 31329
21:06:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:56.087873 31339 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:58.207091 31329 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.121s user 0.399s sys 0.592s
21:06:58.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:58.207495 31329 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.121s user 0.399s sys 0.592s
21:06:58.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:58.214699 31343 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:58.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.214718 31329 server_base.cc:1047] running on GCE node
21:06:58.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.216041 31329 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:06:58.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.218393 31329 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:06:58.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.219799 31329 hybrid_clock.cc:648] HybridClock initialized: now 1765746418219744 us; error 38 us; skew 500 ppm
21:06:58.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.220834 31329 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:06:58.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:58 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746418, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:06:58.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.227733 31329 init.cc:377] Logged in from keytab as kudu/127.30.67.1@KRBTEST.COM (short username kudu)
21:06:58.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.232939 31329 webserver.cc:492] Webserver started at http://127.30.67.1:42683/ using document root <none> and password file <none>
21:06:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.234051 31329 fs_manager.cc:362] Metadata directory not provided
21:06:58.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.234306 31329 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:06:58.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.234833 31329 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:06:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.240664 31329 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/data/instance:
21:06:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "119012e628af42b1b996a5983d9cd6b4"
21:06:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:58 on dist-test-slave-x0xn"
21:06:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.242043 31329 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/wal/instance:
21:06:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "119012e628af42b1b996a5983d9cd6b4"
21:06:58.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:58 on dist-test-slave-x0xn"
21:06:58.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.258289 31329 fs_manager.cc:696] Time spent creating directory manager: real 0.016s user 0.013s sys 0.001s
21:06:58.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.289157 31351 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:06:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.290485 31329 fs_manager.cc:730] Time spent opening block manager: real 0.028s user 0.009s sys 0.017s
21:06:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.291453 31329 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/wal
21:06:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "119012e628af42b1b996a5983d9cd6b4"
21:06:58.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:58 on dist-test-slave-x0xn"
21:06:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.291868 31329 fs_report.cc:389] FS layout report
21:06:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:06:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/wal
21:06:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/wal
21:06:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/data/data
21:06:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:06:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:06:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:06:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:06:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:06:58.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:06:58.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:06:58.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:06:58.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:06:58.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:06:58.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:06:58.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.359074 31329 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:06:58.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.369475 31329 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:06:58.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.370245 31329 kserver.cc:163] Server-wide thread pool size limit: 3276
21:06:58.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.391165 31329 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:06:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.402729 31329 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:06:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.403172 31329 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:06:58.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.403575 31329 ts_tablet_manager.cc:616] Registered 0 tablets
21:06:58.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.403883 31329 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:06:58.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.600925 31329 rpc_server.cc:307] RPC server started. Bound to: 127.30.67.1:42481
21:06:58.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.601058 31464 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.67.1:42481 every 8 connection(s)
21:06:58.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.604598 31329 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/data/info.pb
21:06:58.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.612514 30988 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 31329
21:06:58.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.613147 30988 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-0/wal/instance
21:06:58.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:58 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746418, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.1@KRBTEST.COM for kudu/127.30.67.60@KRBTEST.COM
21:06:58.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:58 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746418, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.1@KRBTEST.COM for kudu/127.30.67.61@KRBTEST.COM
21:06:58.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.67.2@KRBTEST.COM; defaulting to no policy
21:06:58.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:58 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746418, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.1@KRBTEST.COM for kudu/127.30.67.62@KRBTEST.COM
21:06:58.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.688346 31465 heartbeater.cc:344] Connected to a master server at 127.30.67.60:41497
21:06:58.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.688833 31465 heartbeater.cc:461] Registering TS with master...
21:06:58.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.690083 31465 heartbeater.cc:507] Master 127.30.67.60:41497 requested a full tablet report, sending...
21:06:58.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.693421 31221 ts_manager.cc:194] Registered new tserver with Master: 119012e628af42b1b996a5983d9cd6b4 (127.30.67.1:42481)
21:06:58.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:58.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.67.2@KRBTEST.COM" created.
21:06:58.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.698729 31466 heartbeater.cc:344] Connected to a master server at 127.30.67.61:41887
21:06:58.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.699066 31466 heartbeater.cc:461] Registering TS with master...
21:06:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.699939 31466 heartbeater.cc:507] Master 127.30.67.61:41887 requested a full tablet report, sending...
21:06:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.703301 31135 ts_manager.cc:194] Registered new tserver with Master: 119012e628af42b1b996a5983d9cd6b4 (127.30.67.1:42481)
21:06:58.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.705767 31135 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.30.67.1@KRBTEST.COM'} at 127.30.67.1:52973
21:06:58.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.709544 31469 heartbeater.cc:344] Connected to a master server at 127.30.67.62:45687
21:06:58.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.709863 31469 heartbeater.cc:461] Registering TS with master...
21:06:58.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.710460 31469 heartbeater.cc:507] Master 127.30.67.62:45687 requested a full tablet report, sending...
21:06:58.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.712749 31052 ts_manager.cc:194] Registered new tserver with Master: 119012e628af42b1b996a5983d9cd6b4 (127.30.67.1:42481)
21:06:58.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:58.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:58.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:58.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.30.67.2@KRBTEST.COM; defaulting to no policy
21:06:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:58.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.30.67.2@KRBTEST.COM" created.
21:06:58.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:06:58.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.2 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:58.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.2 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:06:58.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:58.753556 30988 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:58.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/wal
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/data
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/logs
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/data/info.pb
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.2:0
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.67.2
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.2
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.2
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:58.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:06:59.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:59.064411 31479 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:06:59.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:59.064994 31479 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:06:59.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:59.065938 31479 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:06:59.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:59.112540 31479 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:06:59.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:59.113518 31479 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.67.2
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:59.148044 31479 tablet_server_runner.cc:78] Tablet server non-default flags:
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/data
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/wal
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.2
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.2:0
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:06:59.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/data/info.pb
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.2
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.67.2
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/logs
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:06:59.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:59.149199 31479 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:06:59.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:59.150115 31479 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:06:59.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:59.162789 31486 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:06:59.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:59.710014 31466 heartbeater.cc:499] Master 127.30.67.61:41887 was elected leader, sending a full tablet report...
21:06:59.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:06:59 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746419, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:06:59.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:06:59.734629 31031 init.cc:295] Successfully reacquired a new kerberos TGT
21:07:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:06:59.163872 31487 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:00.565176 31485 debug-util.cc:398] Leaking SignalData structure 0x7b08000282a0 after lost signal to thread 31479
21:07:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:00.642105 31479 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.479s user 0.532s sys 0.866s
21:07:00.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:00.643543 31479 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.480s user 0.532s sys 0.866s
21:07:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:00.644061 31488 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1479 milliseconds
21:07:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:00.644542 31489 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:00.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.644615 31479 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:00.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.648326 31479 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:00.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.650197 31479 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:00.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.651544 31479 hybrid_clock.cc:648] HybridClock initialized: now 1765746420651494 us; error 38 us; skew 500 ppm
21:07:00.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.652513 31479 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:00.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:00 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746420, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:00.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.658257 31479 init.cc:377] Logged in from keytab as kudu/127.30.67.2@KRBTEST.COM (short username kudu)
21:07:00.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.662969 31479 webserver.cc:492] Webserver started at http://127.30.67.2:45253/ using document root <none> and password file <none>
21:07:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.663828 31479 fs_manager.cc:362] Metadata directory not provided
21:07:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.664036 31479 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:00.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.664515 31479 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:07:00.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.668505 31479 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/data/instance:
21:07:00.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a3601b0e98e4444996443eda88e9b32"
21:07:00.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:07:00 on dist-test-slave-x0xn"
21:07:00.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.669550 31479 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/wal/instance:
21:07:00.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a3601b0e98e4444996443eda88e9b32"
21:07:00.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:07:00 on dist-test-slave-x0xn"
21:07:00.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.676262 31479 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.002s
21:07:00.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.681180 31498 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:00.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.682116 31479 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
21:07:00.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.682372 31479 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/wal
21:07:00.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a3601b0e98e4444996443eda88e9b32"
21:07:00.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:07:00 on dist-test-slave-x0xn"
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.682662 31479 fs_report.cc:389] FS layout report
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/wal
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/wal
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/data/data
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:00.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:00.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.730911 31479 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.736007 31479 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:00.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.736429 31479 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:00.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.738794 31479 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:07:00.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.742806 31479 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:07:00.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.742993 31479 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:07:00.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.743182 31479 ts_tablet_manager.cc:616] Registered 0 tablets
21:07:00.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.743314 31479 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:07:00.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.873303 31479 rpc_server.cc:307] RPC server started. Bound to: 127.30.67.2:35437
21:07:00.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.873368 31610 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.67.2:35437 every 8 connection(s)
21:07:00.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.875883 31479 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/data/info.pb
21:07:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.886365 30988 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 31479
21:07:00.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.886855 30988 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-1/wal/instance
21:07:00.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:00 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746420, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.2@KRBTEST.COM for kudu/127.30.67.60@KRBTEST.COM
21:07:00.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:00 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746420, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.2@KRBTEST.COM for kudu/127.30.67.62@KRBTEST.COM
21:07:00.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:00 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746420, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.2@KRBTEST.COM for kudu/127.30.67.61@KRBTEST.COM
21:07:00.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.30.67.3@KRBTEST.COM; defaulting to no policy
21:07:00.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:00.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.30.67.3@KRBTEST.COM" created.
21:07:00.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.936978 31612 heartbeater.cc:344] Connected to a master server at 127.30.67.61:41887
21:07:00.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.937479 31612 heartbeater.cc:461] Registering TS with master...
21:07:00.939 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.938757 31612 heartbeater.cc:507] Master 127.30.67.61:41887 requested a full tablet report, sending...
21:07:00.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.940585 31613 heartbeater.cc:344] Connected to a master server at 127.30.67.62:45687
21:07:00.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.940927 31613 heartbeater.cc:461] Registering TS with master...
21:07:00.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.941617 31611 heartbeater.cc:344] Connected to a master server at 127.30.67.60:41497
21:07:00.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.941977 31611 heartbeater.cc:461] Registering TS with master...
21:07:00.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.942852 31135 ts_manager.cc:194] Registered new tserver with Master: 2a3601b0e98e4444996443eda88e9b32 (127.30.67.2:35437)
21:07:00.945 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.944589 31135 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.30.67.2@KRBTEST.COM'} at 127.30.67.2:52923
21:07:00.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.946077 31613 heartbeater.cc:507] Master 127.30.67.62:45687 requested a full tablet report, sending...
21:07:00.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.950496 31611 heartbeater.cc:507] Master 127.30.67.60:41497 requested a full tablet report, sending...
21:07:00.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.952860 31221 ts_manager.cc:194] Registered new tserver with Master: 2a3601b0e98e4444996443eda88e9b32 (127.30.67.2:35437)
21:07:00.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:00.952943 31052 ts_manager.cc:194] Registered new tserver with Master: 2a3601b0e98e4444996443eda88e9b32 (127.30.67.2:35437)
21:07:00.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:00.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:07:00.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.30.67.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:07:00.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.30.67.3@KRBTEST.COM; defaulting to no policy
21:07:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:00.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.30.67.3@KRBTEST.COM" created.
21:07:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
21:07:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.3 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:07:01.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.30.67.3 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab.
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:01.002291 30988 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/wal
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/data
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/logs
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/data/info.pb
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:01.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.3:0
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.67.3
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.3
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.3
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:01.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:01.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:01.293251 31626 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:01.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:01.293710 31626 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:01.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:01.294332 31626 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:01.326828 31626 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:01.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:01.327736 31626 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.30.67.3
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:01.363936 31626 tablet_server_runner.cc:78] Tablet server non-default flags:
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/data
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/wal
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.3
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.3:0
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/data/info.pb
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.3
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.30.67.3
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/logs
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:01.364 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
21:07:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:07:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:07:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:07:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:01.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:01.365175 31626 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:01.366200 31626 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:01.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:01.378219 31634 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:01.509621 31166 debug-util.cc:398] Leaking SignalData structure 0x7b080007dc00 after lost signal to thread 31097
21:07:01.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:01 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746421, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:01.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:01.889400 31114 init.cc:295] Successfully reacquired a new kerberos TGT
21:07:01.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:01.948500 31612 heartbeater.cc:499] Master 127.30.67.61:41887 was elected leader, sending a full tablet report...
21:07:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:02.659688 31636 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1280 milliseconds
21:07:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:01.380685 31635 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:02.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:02.661736 31637 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:02.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:02.664626 31626 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.286s user 0.416s sys 0.825s
21:07:02.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:02.664953 31626 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.287s user 0.416s sys 0.825s
21:07:02.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.665201 31626 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:02.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.678251 31626 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.680996 31626 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:02.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.682422 31626 hybrid_clock.cc:648] HybridClock initialized: now 1765746422682302 us; error 132 us; skew 500 ppm
21:07:02.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.683466 31626 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:02.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:02 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746422, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:02.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.692373 31626 init.cc:377] Logged in from keytab as kudu/127.30.67.3@KRBTEST.COM (short username kudu)
21:07:02.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.701041 31626 webserver.cc:492] Webserver started at http://127.30.67.3:41977/ using document root <none> and password file <none>
21:07:02.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.702201 31626 fs_manager.cc:362] Metadata directory not provided
21:07:02.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.702459 31626 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:02.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.702972 31626 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
21:07:02.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.709087 31626 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/data/instance:
21:07:02.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83b56e1c85ac4fa994a61275af243401"
21:07:02.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:07:02 on dist-test-slave-x0xn"
21:07:02.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.710484 31626 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/wal/instance:
21:07:02.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83b56e1c85ac4fa994a61275af243401"
21:07:02.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:07:02 on dist-test-slave-x0xn"
21:07:02.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.719956 31626 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.007s sys 0.003s
21:07:02.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.733048 31646 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:02.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.734706 31626 fs_manager.cc:730] Time spent opening block manager: real 0.011s user 0.010s sys 0.003s
21:07:02.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.735124 31626 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/wal
21:07:02.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "83b56e1c85ac4fa994a61275af243401"
21:07:02.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:07:02 on dist-test-slave-x0xn"
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.735536 31626 fs_report.cc:389] FS layout report
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/wal
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/wal
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/data/data
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:02.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:02.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:02.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:02.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:02.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.809334 31626 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:02.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.815524 31626 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:02.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.816068 31626 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:02.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.819406 31626 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
21:07:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.825031 31626 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
21:07:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.825310 31626 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
21:07:02.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.825599 31626 ts_tablet_manager.cc:616] Registered 0 tablets
21:07:02.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:02.825801 31626 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
21:07:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.087463 31626 rpc_server.cc:307] RPC server started. Bound to: 127.30.67.3:39683
21:07:03.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.087618 31758 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.67.3:39683 every 8 connection(s)
21:07:03.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.090960 31626 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/data/info.pb
21:07:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.098364 30988 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 31626
21:07:03.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.098801 30988 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/ts-2/wal/instance
21:07:03.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:03 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746422, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.3@KRBTEST.COM for kudu/127.30.67.60@KRBTEST.COM
21:07:03.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:03 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746422, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.3@KRBTEST.COM for kudu/127.30.67.62@KRBTEST.COM
21:07:03.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:03 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746422, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.3@KRBTEST.COM for kudu/127.30.67.61@KRBTEST.COM
21:07:03.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.176410 31759 heartbeater.cc:344] Connected to a master server at 127.30.67.60:41497
21:07:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.176913 31759 heartbeater.cc:461] Registering TS with master...
21:07:03.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.177976 31759 heartbeater.cc:507] Master 127.30.67.60:41497 requested a full tablet report, sending...
21:07:03.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.180752 31221 ts_manager.cc:194] Registered new tserver with Master: 83b56e1c85ac4fa994a61275af243401 (127.30.67.3:39683)
21:07:03.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.185493 31761 heartbeater.cc:344] Connected to a master server at 127.30.67.62:45687
21:07:03.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.185846 31761 heartbeater.cc:461] Registering TS with master...
21:07:03.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.186731 31761 heartbeater.cc:507] Master 127.30.67.62:45687 requested a full tablet report, sending...
21:07:03.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.189229 31052 ts_manager.cc:194] Registered new tserver with Master: 83b56e1c85ac4fa994a61275af243401 (127.30.67.3:39683)
21:07:03.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.192703 31760 heartbeater.cc:344] Connected to a master server at 127.30.67.61:41887
21:07:03.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.193039 31760 heartbeater.cc:461] Registering TS with master...
21:07:03.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.194233 31760 heartbeater.cc:507] Master 127.30.67.61:41887 requested a full tablet report, sending...
21:07:03.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.196000 31135 ts_manager.cc:194] Registered new tserver with Master: 83b56e1c85ac4fa994a61275af243401 (127.30.67.3:39683)
21:07:03.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.197645 31135 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.30.67.3@KRBTEST.COM'} at 127.30.67.3:34513
21:07:03.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.211010 30988 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
21:07:03.221 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:03.223 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
21:07:03.232 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
21:07:03.233 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
21:07:03.236 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.30.67.62"
port: 45687
}
bound_http_address {
host: "127.30.67.62"
port: 43597
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.30.67.61"
port: 41887
}
bound_http_address {
host: "127.30.67.61"
port: 40011
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.30.67.60"
port: 41497
}
bound_http_address {
host: "127.30.67.60"
port: 41617
}
}
}
21:07:03.236 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
21:07:03.238 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.30.67.1"
port: 42481
}
bound_http_address {
host: "127.30.67.1"
port: 42683
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.30.67.2"
port: 35437
}
bound_http_address {
host: "127.30.67.2"
port: 45253
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.30.67.3"
port: 39683
}
bound_http_address {
host: "127.30.67.3"
port: 41977
}
}
}
21:07:03.238 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:07:03.239 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:07:03.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:03 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746423, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:03.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:07:03.285 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:03.287 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc
21:07:03.294 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:07:03.296 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc
21:07:03.297 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:07:03.301 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
21:07:03.303 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
21:07:03.303 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
21:07:03.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:03 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746423, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:03.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
21:07:03.338 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:03.339 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc
21:07:03.340 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
21:07:03.341 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.67.61:41887
21:07:03.343 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
21:07:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.343613 30988 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 31096
21:07:03.364 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:03.365 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.67.62:45687
21:07:03.365 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
21:07:03.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.365592 30988 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 31017
21:07:03.386 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:03.386 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.30.67.60:41497
21:07:03.387 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
21:07:03.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.387269 30988 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 31180
21:07:03.407 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:03.408 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.67.61:41887
21:07:03.412 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.414081 30988 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/logs
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data/info.pb
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:03.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.61:41887
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.61
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40011
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.61
21:07:03.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:07:03.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.715689 31774 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:03.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.716682 31774 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.716718 31465 heartbeater.cc:646] Failed to heartbeat to 127.30.67.60:41497 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:07:03.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.717581 31774 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:03.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.749585 31774 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:07:03.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.749788 31774 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:07:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.750046 31774 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.750237 31774 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.750425 31774 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:07:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.750671 31774 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:03.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.750819 31774 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37431/jwks.json
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.786233 31774 master_runner.cc:386] Master server non-default flags:
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.61
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.61:41887
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:03.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data/info.pb
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.61
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40011
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/logs
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:03.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.787371 31774 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:03.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:03.788333 31774 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:03.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.798484 31782 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:03.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.971210 31613 heartbeater.cc:646] Failed to heartbeat to 127.30.67.62:45687 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.30.67.62:45687: connect: Connection refused (error 111)
21:07:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:04.186743 31759 heartbeater.cc:646] Failed to heartbeat to 127.30.67.60:41497 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:07:04.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:04.200414 31760 heartbeater.cc:499] Master 127.30.67.61:41887 was elected leader, sending a full tablet report...
21:07:05.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:03.799324 31783 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:05.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.053773 31784 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1252 milliseconds
21:07:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.055248 31785 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:05.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.055416 31774 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:05.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.056704 31774 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:05.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.059058 31774 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.060372 31774 hybrid_clock.cc:648] HybridClock initialized: now 1765746425060337 us; error 41 us; skew 500 ppm
21:07:05.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.061152 31774 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:05 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746425, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.61@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.066830 31774 init.cc:377] Logged in from keytab as kudu/127.30.67.61@KRBTEST.COM (short username kudu)
21:07:05.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.071610 31774 webserver.cc:492] Webserver started at http://127.30.67.61:40011/ using document root <none> and password file <none>
21:07:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.072502 31774 fs_manager.cc:362] Metadata directory not provided
21:07:05.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.072697 31774 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:05.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.080083 31774 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.007s sys 0.000s
21:07:05.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.084604 31800 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:05.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.085723 31774 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
21:07:05.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.086067 31774 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal
21:07:05.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "73433c1d8df9487bb1424ca34764d4db"
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:51 on dist-test-slave-x0xn"
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.087881 31774 fs_report.cc:389] FS layout report
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/wal
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data/data
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:05.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:05.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:05.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:05.143 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.142793 31774 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.148343 31774 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:05.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.148739 31774 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:05.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.217974 31774 rpc_server.cc:307] RPC server started. Bound to: 127.30.67.61:41887
21:07:05.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.218025 31853 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.67.61:41887 every 8 connection(s)
21:07:05.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.220934 31774 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-1/data/info.pb
21:07:05.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.223820 30988 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 31774
21:07:05.224 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:05.224 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.67.62:45687
21:07:05.225 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.226711 30988 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/logs
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data/info.pb
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:05.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.62:45687
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.62
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43597
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.62
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:05.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.232441 31854 sys_catalog.cc:263] Verifying existing consensus state
21:07:05.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.239959 31854 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: Bootstrap starting.
21:07:05.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.290583 31854 log.cc:826] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: Log is configured to *not* fsync() on all Append() calls
21:07:05.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.304947 31854 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:07:05.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.305683 31854 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: Bootstrap complete.
21:07:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.327158 31854 raft_consensus.cc:359] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:07:05.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.329293 31854 raft_consensus.cc:740] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 73433c1d8df9487bb1424ca34764d4db, State: Initialized, Role: FOLLOWER
21:07:05.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.330121 31854 consensus_queue.cc:260] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:07:05.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.333014 31858 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:07:05.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.333962 31858 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: This master's current role is: FOLLOWER
21:07:05.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.336676 31854 sys_catalog.cc:565] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: configured and running, proceeding with master startup.
21:07:05.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.369256 31869 catalog_manager.cc:1269] Loaded cluster ID: a48b2d3c23b5424ea5338a0d99b73d85
21:07:05.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.369814 31869 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: loading cluster ID for follower catalog manager: success
21:07:05.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.377856 31869 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: acquiring CA information for follower catalog manager: success
21:07:05.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.384277 31869 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:07:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.366729 31774 master_runner.cc:186] Error getting master registration for 127.30.67.62:45687: Network error: Client connection negotiation failed: client connection to 127.30.67.62:45687: connect: Connection refused (error 111),
21:07:05.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.390193 31774 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:07:05.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.617393 31856 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:05.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.617882 31856 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:05.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.618572 31856 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:05.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.650105 31856 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:07:05.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.650309 31856 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:07:05.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.650564 31856 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:05.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.650758 31856 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.650954 31856 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:07:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.651136 31856 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:05.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.651276 31856 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37431/jwks.json
21:07:05.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.677739 31607 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba9a0 after lost signal to thread 31480
21:07:05.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.678795 31607 debug-util.cc:398] Leaking SignalData structure 0x7b08000baec0 after lost signal to thread 31610
21:07:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.692691 31856 master_runner.cc:386] Master server non-default flags:
21:07:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:07:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data
21:07:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal
21:07:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:07:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:05.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.62
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.62:45687
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data/info.pb
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.62
21:07:05.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43597
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/logs
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:07:05.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:07:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.694240 31856 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:05.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.695519 31856 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:05.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.742966 31879 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:05.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.780961 31466 heartbeater.cc:344] Connected to a master server at 127.30.67.61:41887
21:07:05.787 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.787281 31817 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "119012e628af42b1b996a5983d9cd6b4" instance_seqno: 1765746418562810) as {username='kudu', principal='kudu/127.30.67.1@KRBTEST.COM'} at 127.30.67.1:44593; Asking this server to re-register.
21:07:05.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.788604 31466 heartbeater.cc:461] Registering TS with master...
21:07:05.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:05.791689 31817 ts_manager.cc:194] Registered new tserver with Master: 119012e628af42b1b996a5983d9cd6b4 (127.30.67.1:42481)
21:07:06.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:06.033000 31612 heartbeater.cc:344] Connected to a master server at 127.30.67.61:41887
21:07:06.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:06.035725 31817 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2a3601b0e98e4444996443eda88e9b32" instance_seqno: 1765746420842938) as {username='kudu', principal='kudu/127.30.67.2@KRBTEST.COM'} at 127.30.67.2:57071; Asking this server to re-register.
21:07:06.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:06.037312 31612 heartbeater.cc:461] Registering TS with master...
21:07:06.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:06.040694 31817 ts_manager.cc:194] Registered new tserver with Master: 2a3601b0e98e4444996443eda88e9b32 (127.30.67.2:35437)
21:07:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:06.286630 31760 heartbeater.cc:344] Connected to a master server at 127.30.67.61:41887
21:07:06.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:06.287187 31760 heartbeater.cc:499] Master 127.30.67.61:41887 was elected leader, sending a full tablet report...
21:07:06.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:06.290074 31817 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "83b56e1c85ac4fa994a61275af243401" instance_seqno: 1765746423035589) as {username='kudu', principal='kudu/127.30.67.3@KRBTEST.COM'} at 127.30.67.3:41569; Asking this server to re-register.
21:07:06.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:06.291879 31760 heartbeater.cc:461] Registering TS with master...
21:07:06.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:06.295418 31817 ts_manager.cc:194] Registered new tserver with Master: 83b56e1c85ac4fa994a61275af243401 (127.30.67.3:39683)
21:07:07.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.135823 31893 raft_consensus.cc:493] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:07:07.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.138087 31893 raft_consensus.cc:515] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:07:07.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.162930 31893 leader_election.cc:290] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 422367ec3d01457bb1fb5b6a151edb46 (127.30.67.62:45687), 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497)
21:07:07.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.176023 31801 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:07:07.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.192432 31802 leader_election.cc:336] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 422367ec3d01457bb1fb5b6a151edb46 (127.30.67.62:45687): Network error: Client connection negotiation failed: client connection to 127.30.67.62:45687: connect: Connection refused (error 111)
21:07:07.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.195950 31801 leader_election.cc:336] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497): Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:07:07.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.196573 31801 leader_election.cc:304] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 73433c1d8df9487bb1424ca34764d4db; no voters: 422367ec3d01457bb1fb5b6a151edb46, 65976ecfedef4b369da0bd68c0ff8356
21:07:07.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.197463 31893 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:07:07.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.119727 31876 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 31856
21:07:07.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.246534 31876 kernel_stack_watchdog.cc:198] Thread 31856 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 377ms:
21:07:07.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
21:07:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
21:07:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
21:07:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
21:07:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.246654 31856 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.506s user 0.499s sys 0.801s
21:07:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:05.749172 31877 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:07.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.247048 31856 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.507s user 0.499s sys 0.801s
21:07:07.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.280596 31856 server_base.cc:1047] running on GCE node
21:07:07.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.288996 31900 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:07.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.290508 31856 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:07.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.293560 31856 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:07.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.294996 31856 hybrid_clock.cc:648] HybridClock initialized: now 1765746427294941 us; error 45 us; skew 500 ppm
21:07:07.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.296027 31856 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:07.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:07 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746427, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.62@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:07.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.308097 31856 init.cc:377] Logged in from keytab as kudu/127.30.67.62@KRBTEST.COM (short username kudu)
21:07:07.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.314548 31856 webserver.cc:492] Webserver started at http://127.30.67.62:43597/ using document root <none> and password file <none>
21:07:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.315434 31856 fs_manager.cc:362] Metadata directory not provided
21:07:07.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.315665 31856 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:07.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.339461 31856 fs_manager.cc:714] Time spent opening directory manager: real 0.020s user 0.018s sys 0.003s
21:07:07.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.344715 31906 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:07.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.345877 31856 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
21:07:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.346218 31856 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal
21:07:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "422367ec3d01457bb1fb5b6a151edb46"
21:07:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:49 on dist-test-slave-x0xn"
21:07:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.348270 31856 fs_report.cc:389] FS layout report
21:07:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal
21:07:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/wal
21:07:07.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data/data
21:07:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:07.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:07.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:07.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.402957 31856 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:07.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.408316 31856 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:07.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.408773 31856 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:07.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.529719 31856 rpc_server.cc:307] RPC server started. Bound to: 127.30.67.62:45687
21:07:07.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.529848 31957 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.67.62:45687 every 8 connection(s)
21:07:07.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.532773 31856 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-0/data/info.pb
21:07:07.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.533527 30988 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 31856
21:07:07.534 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:07.535 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.30.67.60:41497
21:07:07.535 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
21:07:07.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.538473 30988 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:07:07.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/logs
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data/info.pb
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.60:41497
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.60
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41617
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.60
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
21:07:07.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:07:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:07:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:07:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
21:07:07.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
21:07:07.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.558748 31958 sys_catalog.cc:263] Verifying existing consensus state
21:07:07.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.567713 31958 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: Bootstrap starting.
21:07:07.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.573256 30973 debug-util.cc:398] Leaking SignalData structure 0x7b08000ad080 after lost signal to thread 30847
21:07:07.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.574326 30973 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9a20 after lost signal to thread 30976
21:07:07.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.634348 31958 log.cc:826] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: Log is configured to *not* fsync() on all Append() calls
21:07:07.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.675125 31958 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:07:07.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.676093 31958 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: Bootstrap complete.
21:07:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.749851 31958 raft_consensus.cc:359] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:07:07.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.753082 31958 raft_consensus.cc:740] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 422367ec3d01457bb1fb5b6a151edb46, State: Initialized, Role: FOLLOWER
21:07:07.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.754202 31958 consensus_queue.cc:260] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:07:07.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.763273 31962 sys_catalog.cc:455] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:07:07.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.766976 31962 sys_catalog.cc:458] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: This master's current role is: FOLLOWER
21:07:07.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.770586 31958 sys_catalog.cc:565] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: configured and running, proceeding with master startup.
21:07:07.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.858479 31976 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
21:07:07.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.882649 31469 heartbeater.cc:344] Connected to a master server at 127.30.67.62:45687
21:07:07.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.889092 31923 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "119012e628af42b1b996a5983d9cd6b4" instance_seqno: 1765746418562810) as {username='kudu', principal='kudu/127.30.67.1@KRBTEST.COM'} at 127.30.67.1:57219; Asking this server to re-register.
21:07:07.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.890777 31469 heartbeater.cc:461] Registering TS with master...
21:07:07.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.894380 31922 ts_manager.cc:194] Registered new tserver with Master: 119012e628af42b1b996a5983d9cd6b4 (127.30.67.1:42481)
21:07:07.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:07 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746427, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.62@KRBTEST.COM for kudu/127.30.67.62@KRBTEST.COM
21:07:07.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.925339 30490 debug-util.cc:398] Leaking SignalData structure 0x7b08000b8980 after lost signal to thread 30418
21:07:07.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:07.926064 30490 debug-util.cc:398] Leaking SignalData structure 0x7b080007bba0 after lost signal to thread 30493
21:07:07.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:07 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746427, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.62@KRBTEST.COM for kudu/127.30.67.61@KRBTEST.COM
21:07:08.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.124140 31613 heartbeater.cc:344] Connected to a master server at 127.30.67.62:45687
21:07:08.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.126718 31922 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2a3601b0e98e4444996443eda88e9b32" instance_seqno: 1765746420842938) as {username='kudu', principal='kudu/127.30.67.2@KRBTEST.COM'} at 127.30.67.2:42451; Asking this server to re-register.
21:07:08.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.128242 31613 heartbeater.cc:461] Registering TS with master...
21:07:08.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.130890 31922 ts_manager.cc:194] Registered new tserver with Master: 2a3601b0e98e4444996443eda88e9b32 (127.30.67.2:35437)
21:07:08.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.137972 31959 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
21:07:08.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.138698 31959 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
21:07:08.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.139887 31959 flags.cc:425] Enabled unsafe flag: --never_fsync=true
21:07:08.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:07.991875 31856 master_runner.cc:186] Error getting master registration for 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111),
21:07:08.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.152313 31856 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:07:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.201680 31959 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
21:07:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.201990 31959 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
21:07:08.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.202507 31959 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
21:07:08.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.202868 31959 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
21:07:08.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.203207 31959 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
21:07:08.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.203537 31959 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
21:07:08.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.203804 31959 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:37431/jwks.json
21:07:08.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:08 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746428, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.1@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:08.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.232347 31346 init.cc:295] Successfully reacquired a new kerberos TGT
21:07:08.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.340461 31761 heartbeater.cc:344] Connected to a master server at 127.30.67.62:45687
21:07:08.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.343060 31922 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "83b56e1c85ac4fa994a61275af243401" instance_seqno: 1765746423035589) as {username='kudu', principal='kudu/127.30.67.3@KRBTEST.COM'} at 127.30.67.3:41893; Asking this server to re-register.
21:07:08.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.344245 31761 heartbeater.cc:461] Registering TS with master...
21:07:08.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.346912 31922 ts_manager.cc:194] Registered new tserver with Master: 83b56e1c85ac4fa994a61275af243401 (127.30.67.3:39683)
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.378726 31959 master_runner.cc:386] Master server non-default flags:
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.30.67.20:41821
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.30.67.62:45687,127.30.67.61:41887,127.30.67.60:41497
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/krb5kdc/kudu.keytab
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.30.67.60
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.30.67.60:41497
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:37431/jwks.json
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
21:07:08.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data/info.pb
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.30.67.60
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=41617
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/logs
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision dc741732bb6a40213e11a13869fad06a49a209a3
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 14 Dec 2025 20:43:22 UTC on 5fd53c4cbb9d
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 9517
21:07:08.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
21:07:08.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.382196 31959 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
21:07:08.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.384454 31959 file_cache.cc:492] Constructed file cache file cache with capacity 419430
21:07:08.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.416843 31990 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:08.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.875993 31976 catalog_manager.cc:1269] Loaded cluster ID: a48b2d3c23b5424ea5338a0d99b73d85
21:07:08.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.876286 31976 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: loading cluster ID for follower catalog manager: success
21:07:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.882965 31976 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: acquiring CA information for follower catalog manager: success
21:07:08.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:08.892058 31976 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:07:09.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.162607 32000 raft_consensus.cc:493] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:07:09.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.167994 32000 raft_consensus.cc:515] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:07:09.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.201694 32000 leader_election.cc:290] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 422367ec3d01457bb1fb5b6a151edb46 (127.30.67.62:45687), 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497)
21:07:09.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.203354 32003 raft_consensus.cc:493] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
21:07:09.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.203922 32003 raft_consensus.cc:515] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:07:09.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.218394 31933 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "73433c1d8df9487bb1424ca34764d4db" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "422367ec3d01457bb1fb5b6a151edb46" is_pre_election: true
21:07:09.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.219184 31933 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 73433c1d8df9487bb1424ca34764d4db in term 1.
21:07:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.220554 31802 leader_election.cc:304] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 422367ec3d01457bb1fb5b6a151edb46, 73433c1d8df9487bb1424ca34764d4db; no voters:
21:07:09.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.221621 32000 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Leader pre-election won for term 2
21:07:09.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.222093 32000 raft_consensus.cc:493] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
21:07:09.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.223404 32000 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 1 FOLLOWER]: Advancing to term 2
21:07:09.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.233227 31827 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "422367ec3d01457bb1fb5b6a151edb46" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "73433c1d8df9487bb1424ca34764d4db" is_pre_election: true
21:07:09.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:09.234258 31801 leader_election.cc:336] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497): Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:07:09.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:09.248180 31907 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:07:09.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.252220 32003 leader_election.cc:290] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 73433c1d8df9487bb1424ca34764d4db (127.30.67.61:41887), 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497)
21:07:09.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:09.255358 31907 leader_election.cc:336] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497): Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:07:09.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.240682 32000 raft_consensus.cc:515] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:07:09.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.265820 31827 raft_consensus.cc:2393] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 422367ec3d01457bb1fb5b6a151edb46 in current term 2: Already voted for candidate 73433c1d8df9487bb1424ca34764d4db in this term.
21:07:09.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.272035 31909 leader_election.cc:304] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 422367ec3d01457bb1fb5b6a151edb46; no voters: 65976ecfedef4b369da0bd68c0ff8356, 73433c1d8df9487bb1424ca34764d4db
21:07:09.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.273649 32003 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 1 FOLLOWER]: Advancing to term 2
21:07:09.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.275784 31933 tablet_service.cc:1911] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "73433c1d8df9487bb1424ca34764d4db" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "422367ec3d01457bb1fb5b6a151edb46"
21:07:09.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:09.281601 31801 leader_election.cc:336] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497): Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111)
21:07:09.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.282392 32000 leader_election.cc:290] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 2 election: Requested vote from peers 422367ec3d01457bb1fb5b6a151edb46 (127.30.67.62:45687), 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497)
21:07:09.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.299780 32003 raft_consensus.cc:2749] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
21:07:09.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.306651 31933 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 73433c1d8df9487bb1424ca34764d4db in term 2.
21:07:09.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.312729 31802 leader_election.cc:304] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 422367ec3d01457bb1fb5b6a151edb46, 73433c1d8df9487bb1424ca34764d4db; no voters: 65976ecfedef4b369da0bd68c0ff8356
21:07:09.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.316907 32000 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 2 FOLLOWER]: Leader election won for term 2
21:07:09.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.332252 32000 raft_consensus.cc:697] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [term 2 LEADER]: Becoming Leader. State: Replica: 73433c1d8df9487bb1424ca34764d4db, State: Running, Role: LEADER
21:07:09.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.333407 32000 consensus_queue.cc:237] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:07:09.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.355020 32008 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: SysCatalogTable state changed. Reason: New leader 73433c1d8df9487bb1424ca34764d4db. Latest consensus state: current_term: 2 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:07:09.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.355777 32008 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: This master's current role is: LEADER
21:07:09.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.374128 32010 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
21:07:09.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.393885 32010 catalog_manager.cc:1494] Initializing Kudu cluster ID...
21:07:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.397763 32010 catalog_manager.cc:1269] Loaded cluster ID: a48b2d3c23b5424ea5338a0d99b73d85
21:07:09.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.398036 32010 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
21:07:09.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.402336 32010 catalog_manager.cc:1514] Loading token signing keys...
21:07:09.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.411389 32010 catalog_manager.cc:6038] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db: Loaded TSK: 0
21:07:09.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.412698 32010 catalog_manager.cc:1524] Initializing in-progress tserver states...
21:07:09.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:09.908955 31801 consensus_peers.cc:597] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db -> Peer 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497): Couldn't send request to peer 65976ecfedef4b369da0bd68c0ff8356. Status: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
21:07:09.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.924917 31933 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [term 2 FOLLOWER]: Refusing update from remote peer 73433c1d8df9487bb1424ca34764d4db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
21:07:09.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.926445 32008 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [LEADER]: Connected to new peer: Peer: permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
21:07:09.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.963315 32014 sys_catalog.cc:455] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 73433c1d8df9487bb1424ca34764d4db. Latest consensus state: current_term: 2 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:07:09.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.963971 32014 sys_catalog.cc:458] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: This master's current role is: FOLLOWER
21:07:09.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.991154 32008 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:07:09.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.991818 32008 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: This master's current role is: LEADER
21:07:09.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.999029 32014 sys_catalog.cc:455] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:07:09.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:09.999653 32014 sys_catalog.cc:458] T 00000000000000000000000000000000 P 422367ec3d01457bb1fb5b6a151edb46 [sys.catalog]: This master's current role is: FOLLOWER
21:07:10.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:10.427364 31774 master_runner.cc:186] Error getting master registration for 127.30.67.60:41497: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111),
21:07:10.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:10.427836 31774 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
21:07:10.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:10 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746430, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.2@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:10.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:10.663482 31494 init.cc:295] Successfully reacquired a new kerberos TGT
21:07:10.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:10.811355 31466 heartbeater.cc:499] Master 127.30.67.61:41887 was elected leader, sending a full tablet report...
21:07:11.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:11.055395 31612 heartbeater.cc:499] Master 127.30.67.61:41887 was elected leader, sending a full tablet report...
21:07:11.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:11.328671 31760 heartbeater.cc:499] Master 127.30.67.61:41887 was elected leader, sending a full tablet report...
21:07:12.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:09.806695 31989 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 31959
21:07:12.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:08.428583 31991 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:12.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:12.071151 31959 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.643s user 1.181s sys 1.816s
21:07:12.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:12.072189 31959 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.644s user 1.182s sys 1.816s
21:07:12.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:12.071816 31992 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3644 milliseconds
21:07:12.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.073549 31959 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
21:07:12.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:12.073606 31994 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
21:07:12.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.077906 31959 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
21:07:12.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.080890 31959 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
21:07:12.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.082305 31959 hybrid_clock.cc:648] HybridClock initialized: now 1765746432082213 us; error 85 us; skew 500 ppm
21:07:12.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.083451 31959 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
21:07:12.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:12 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746432, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.60@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:12.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.091301 31959 init.cc:377] Logged in from keytab as kudu/127.30.67.60@KRBTEST.COM (short username kudu)
21:07:12.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.104439 31959 webserver.cc:492] Webserver started at http://127.30.67.60:41617/ using document root <none> and password file <none>
21:07:12.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.105573 31959 fs_manager.cc:362] Metadata directory not provided
21:07:12.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.105861 31959 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
21:07:12.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.115650 31959 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.005s sys 0.001s
21:07:12.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.120728 32042 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
21:07:12.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.121768 31959 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.000s sys 0.004s
21:07:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.122105 31959 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data,/tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal
21:07:12.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "65976ecfedef4b369da0bd68c0ff8356"
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-12-14 21:06:54 on dist-test-slave-x0xn"
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.123903 31959 fs_report.cc:389] FS layout report
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/wal
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data/data
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
21:07:12.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
21:07:12.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.178157 31959 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
21:07:12.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.188855 31959 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
21:07:12.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.189258 31959 kserver.cc:163] Server-wide thread pool size limit: 3276
21:07:12.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:12.314600 31801 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111) [suppressed 8 similar messages]
21:07:12.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:12.317481 31801 consensus_peers.cc:597] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db -> Peer 65976ecfedef4b369da0bd68c0ff8356 (127.30.67.60:41497): Couldn't send request to peer 65976ecfedef4b369da0bd68c0ff8356. Status: Network error: Client connection negotiation failed: client connection to 127.30.67.60:41497: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
21:07:12.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.345674 31959 rpc_server.cc:307] RPC server started. Bound to: 127.30.67.60:41497
21:07:12.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.345813 32093 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.30.67.60:41497 every 8 connection(s)
21:07:12.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.349131 31959 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskNTuEsu/test-tmp/mini-kudu-cluster5402085431849355776/master-2/data/info.pb
21:07:12.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.351707 30988 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu as pid 31959
21:07:12.352 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
21:07:12.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.383782 32094 sys_catalog.cc:263] Verifying existing consensus state
21:07:12.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.392226 32094 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: Bootstrap starting.
21:07:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:12.400769 32098 negotiation.cc:337] Failed RPC negotiation. Trace:
21:07:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.359160 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.30.67.3:45843 (local address 127.30.67.60:41497)
21:07:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.370198 (+ 11038us) server_negotiation.cc:207] Beginning negotiation
21:07:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.370234 (+ 36us) server_negotiation.cc:400] Waiting for connection header
21:07:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.385259 (+ 15025us) server_negotiation.cc:408] Connection header received
21:07:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.391180 (+ 5921us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:07:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.391233 (+ 53us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:07:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.391778 (+ 545us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:07:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.396584 (+ 4806us) server_negotiation.cc:227] Negotiated authn=SASL
21:07:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.398570 (+ 1986us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.30.67.3:45843: BlockingRecv error: recv got EOF from 127.30.67.3:45843 (error 108)
21:07:12.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":10339,"thread_start_us":10804,"threads_started":1}
21:07:12.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.445484 32094 log.cc:826] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: Log is configured to *not* fsync() on all Append() calls
21:07:12.445 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.67.61:41887(127.30.67.61:41887)] connecting to peer
21:07:12.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.464888 32094 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
21:07:12.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.465839 32094 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: Bootstrap complete.
21:07:12.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.495803 32094 raft_consensus.cc:359] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:07:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.498890 32094 raft_consensus.cc:740] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 65976ecfedef4b369da0bd68c0ff8356, State: Initialized, Role: FOLLOWER
21:07:12.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.499796 32094 consensus_queue.cc:260] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } }
21:07:12.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.510381 32094 sys_catalog.cc:565] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: configured and running, proceeding with master startup.
21:07:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.514778 32100 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:07:12.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.515522 32100 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: This master's current role is: FOLLOWER
21:07:12.531 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.67.62:45687(127.30.67.62:45687)] connecting to peer
21:07:12.535 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.30.67.60:41497(127.30.67.60:41497)] connecting to peer
21:07:12.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.607407 32113 catalog_manager.cc:1269] Loaded cluster ID: a48b2d3c23b5424ea5338a0d99b73d85
21:07:12.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.607651 32113 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: loading cluster ID for follower catalog manager: success
21:07:12.611 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.30.67.62:45687(127.30.67.62:45687)] Successfully connected to peer
21:07:12.614 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.30.67.61:41887(127.30.67.61:41887)] Successfully connected to peer
21:07:12.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:12 dist-test-slave-x0xn krb5kdc[31000](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746432, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.60@KRBTEST.COM for kudu/127.30.67.62@KRBTEST.COM
21:07:12.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.632297 32113 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: acquiring CA information for follower catalog manager: success
21:07:12.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.653662 32113 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
21:07:12.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Dec 14 21:07:12 dist-test-slave-x0xn krb5kdc[31000](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1765746432, etypes {rep=17 tkt=17 ses=17}, kudu/127.30.67.3@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
21:07:12.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.699545 31642 init.cc:295] Successfully reacquired a new kerberos TGT
21:07:12.701 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.30.67.60:41497(127.30.67.60:41497)] Successfully connected to peer
21:07:12.743 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
21:07:12.758 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.30.67.61
21:07:12.759 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.30.67.62
21:07:12.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.890774 32069 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [term 1 FOLLOWER]: Advancing to term 2
21:07:12.945 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.30.67.60
21:07:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.968322 31465 heartbeater.cc:344] Connected to a master server at 127.30.67.60:41497
21:07:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.975473 32125 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:07:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.976174 32059 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "119012e628af42b1b996a5983d9cd6b4" instance_seqno: 1765746418562810) as {username='kudu', principal='kudu/127.30.67.1@KRBTEST.COM'} at 127.30.67.1:57267; Asking this server to re-register.
21:07:12.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.976496 32125 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: This master's current role is: FOLLOWER
21:07:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.976980 32100 sys_catalog.cc:455] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 73433c1d8df9487bb1424ca34764d4db. Latest consensus state: current_term: 2 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:07:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.977497 31465 heartbeater.cc:461] Registering TS with master...
21:07:12.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.977581 32100 sys_catalog.cc:458] T 00000000000000000000000000000000 P 65976ecfedef4b369da0bd68c0ff8356 [sys.catalog]: This master's current role is: FOLLOWER
21:07:12.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:12.980756 32059 ts_manager.cc:194] Registered new tserver with Master: 119012e628af42b1b996a5983d9cd6b4 (127.30.67.1:42481)
21:07:13.073 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.30.67.62:45687(127.30.67.62:45687)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:07:13.073 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.67.61:41887(127.30.67.61:41887)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:07:13.080 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.30.67.62:45687: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:07:13.080 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.67.61:41887: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:13.085949 32117 negotiation.cc:337] Failed RPC negotiation. Trace:
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.599439 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:59260 (local address 127.30.67.61:41887)
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.626673 (+ 27234us) server_negotiation.cc:207] Beginning negotiation
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.626702 (+ 29us) server_negotiation.cc:400] Waiting for connection header
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.655629 (+ 28927us) server_negotiation.cc:408] Connection header received
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.699788 (+ 44159us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.699819 (+ 31us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.700041 (+ 222us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.700208 (+ 167us) server_negotiation.cc:227] Negotiated authn=SASL
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.943730 (+243522us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.945681 (+ 1951us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:13.065319 (+119638us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:13.065978 (+ 659us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
21:07:13.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:13.084327 (+ 18349us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:59260: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:59260): Cannot send after transport endpoint shutdown (error 108)
21:07:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":26697,"thread_start_us":26447,"threads_started":1}
21:07:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:13.086412 32118 negotiation.cc:337] Failed RPC negotiation. Trace:
21:07:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.607753 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:35226 (local address 127.30.67.62:45687)
21:07:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.614627 (+ 6874us) server_negotiation.cc:207] Beginning negotiation
21:07:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.614653 (+ 26us) server_negotiation.cc:400] Waiting for connection header
21:07:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.656725 (+ 42072us) server_negotiation.cc:408] Connection header received
21:07:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.700900 (+ 44175us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:07:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.700937 (+ 37us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:07:13.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.701209 (+ 272us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:07:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.701416 (+ 207us) server_negotiation.cc:227] Negotiated authn=SASL
21:07:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.924362 (+222946us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:07:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.926450 (+ 2088us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
21:07:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:13.069454 (+143004us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:07:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:13.069976 (+ 522us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
21:07:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:13.084850 (+ 14874us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:35226: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:35226): Cannot send after transport endpoint shutdown (error 108)
21:07:13.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":6249,"thread_start_us":5962,"threads_started":1}
21:07:13.094 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.30.67.61:41887(127.30.67.61:41887)] handling channelInactive
21:07:13.094 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.30.67.62:45687(127.30.67.62:45687)] handling channelInactive
21:07:13.117 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.30.67.60:41497(127.30.67.60:41497)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:07:13.118 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.30.67.60:41497: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
21:07:13.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.130559 31611 heartbeater.cc:344] Connected to a master server at 127.30.67.60:41497
21:07:13.130 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=769), Traces: [0ms] refreshing cache from master, [56ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.67.61:41887, [156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.67.62:45687, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.67.60:41497, [699ms] Sub RPC ConnectToMaster: received response from server master-127.30.67.61:41887: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [705ms] Sub RPC ConnectToMaster: received response from server master-127.30.67.62:45687: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [742ms] Sub RPC ConnectToMaster: received response from server master-127.30.67.60:41497: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.30.67.61:41887,127.30.67.62:45687,127.30.67.60:41497). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.30.67.61:41887,127.30.67.62:45687,127.30.67.60:41497). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21:07:13.133 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=775), Traces: [0ms] refreshing cache from master, [56ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.67.61:41887, [156ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.67.62:45687, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.30.67.60:41497, [699ms] Sub RPC ConnectToMaster: received response from server master-127.30.67.61:41887: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [705ms] Sub RPC ConnectToMaster: received response from server master-127.30.67.62:45687: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [742ms] Sub RPC ConnectToMaster: received response from server master-127.30.67.60:41497: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
21:07:13.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.133191 32059 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "2a3601b0e98e4444996443eda88e9b32" instance_seqno: 1765746420842938) as {username='kudu', principal='kudu/127.30.67.2@KRBTEST.COM'} at 127.30.67.2:55939; Asking this server to re-register.
21:07:13.137 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.67.61:41887(127.30.67.61:41887)] disconnecting while in state TERMINATED
21:07:13.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.134654 31611 heartbeater.cc:461] Registering TS with master...
21:07:13.138 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.67.62:45687(127.30.67.62:45687)] disconnecting while in state TERMINATED
21:07:13.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.139786 32059 ts_manager.cc:194] Registered new tserver with Master: 2a3601b0e98e4444996443eda88e9b32 (127.30.67.2:35437)
21:07:13.140 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.30.67.60:41497(127.30.67.60:41497)] disconnecting while in state TERMINATED
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:13.147173 32098 negotiation.cc:337] Failed RPC negotiation. Trace:
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.605915 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:57158 (local address 127.30.67.60:41497)
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.607179 (+ 1264us) server_negotiation.cc:207] Beginning negotiation
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.607205 (+ 26us) server_negotiation.cc:400] Waiting for connection header
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.701673 (+ 94468us) server_negotiation.cc:408] Connection header received
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.702254 (+ 581us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.702282 (+ 28us) server_negotiation.cc:462] Received NEGOTIATE request from client
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.702954 (+ 672us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.703103 (+ 149us) server_negotiation.cc:227] Negotiated authn=SASL
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.983115 (+280012us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:12.985761 (+ 2646us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:13.115062 (+129301us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:13.115686 (+ 624us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1214 21:07:13.145937 (+ 30251us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:57158: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:57158): Cannot send after transport endpoint shutdown (error 108)
21:07:13.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":117}
21:07:13.149 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:07:13.152 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.30.67.60:41497(127.30.67.60:41497)] handling channelInactive
21:07:13.153 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:07:13.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.165377 30988 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 31329
21:07:13.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.196736 30988 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 31479
21:07:13.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.234124 30988 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 31626
21:07:13.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.278146 30988 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 31856
21:07:13.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.279636 32031 sys_catalog.cc:455] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "73433c1d8df9487bb1424ca34764d4db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "422367ec3d01457bb1fb5b6a151edb46" member_type: VOTER last_known_addr { host: "127.30.67.62" port: 45687 } } peers { permanent_uuid: "73433c1d8df9487bb1424ca34764d4db" member_type: VOTER last_known_addr { host: "127.30.67.61" port: 41887 } } peers { permanent_uuid: "65976ecfedef4b369da0bd68c0ff8356" member_type: VOTER last_known_addr { host: "127.30.67.60" port: 41497 } } }
21:07:13.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.280671 32031 sys_catalog.cc:458] T 00000000000000000000000000000000 P 73433c1d8df9487bb1424ca34764d4db [sys.catalog]: This master's current role is: LEADER
21:07:13.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.306277 30988 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 31774
21:07:13.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:13.336314 30988 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 31959
21:07:14.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-14T21:07:14Z chronyd exiting
21:07:15.483 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
21:07:15.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:15.484758 30254 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 30539
21:07:15.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:15.504453 30254 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 30704
21:07:15.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:15.527290 30254 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 30842
21:07:15.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:15.550633 30254 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 30266
21:07:15.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:15.573453 30254 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 30337
21:07:15.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20251214 21:07:15.596714 30440 connection.cc:537] client connection to 127.29.139.189:45383 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
21:07:15.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20251214 21:07:15.597103 30254 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskNTuEsu/build/tsan/bin/kudu with pid 30413
21:07:16.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-12-14T21:07:16Z chronyd exiting
21:07:17.672 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.30.67.61:41887,127.30.67.62:45687,127.30.67.60:41497). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.30.67.61:41887,127.30.67.62:45687,127.30.67.60:41497). Exceptions received: [org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more