14:06:58.892 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:06:58.903 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskKowyQv/test-tmp
14:06:58.906 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:06:58.908 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskKowyQv/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:06:59.195 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:06:59.292 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:06:59.293 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:06:59.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:06:59.295905 11865 mini_oidc.cc:150] Starting JWKS server
14:06:59.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:06:59.299155 11865 webserver.cc:480] Webserver started at http://127.0.0.1:41481/ using document root <none> and password file <none>
14:06:59.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:06:59.299561 11865 mini_oidc.cc:186] Starting OIDC Discovery server
14:06:59.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:06:59.301162 11865 webserver.cc:480] Webserver started at http://127.0.0.1:38487/ using document root <none> and password file <none>
14:06:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T14:06:59Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:06:59.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T14:06:59Z Disabled control of system clock
14:06:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:06:59.356348 11865 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:06:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:06:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/wal
14:06:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/data
14:06:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:06:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:06:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:06:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:59.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/logs
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/data/info.pb
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:06:59.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:06:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.126:37365
14:06:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.126
14:06:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:06:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:59.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:59.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:59.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:59.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:06:59.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:59.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41481/jwks.json with env {}
14:06:59.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:06:59.626940 11877 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:06:59.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:06:59.627434 11877 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:06:59.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:06:59.627964 11877 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:06:59.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:06:59.657892 11877 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:06:59.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:06:59.658159 11877 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:06:59.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:06:59.658342 11877 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:06:59.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:06:59.658607 11877 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:06:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:06:59.658790 11877 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:06:59.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:06:59.658942 11877 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41481/jwks.json
14:06:59.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:06:59.693063 11877 master_runner.cc:386] Master server non-default flags:
14:06:59.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:06:59.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:06:59.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:06:59.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:06:59.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/data
14:06:59.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/wal
14:06:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:06:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:06:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:06:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:06:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:06:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.126:37365
14:06:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:06:59.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:06:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:06:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41481/jwks.json
14:06:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:06:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:06:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/data/info.pb
14:06:59.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.126
14:06:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:06:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:06:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:06:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:06:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:06:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:06:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/logs
14:06:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:06:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:06:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:06:59.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:06:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:06:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:06:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:06:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:06:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:06:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:06:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:06:59.694337 11877 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:06:59.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:06:59.695947 11877 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:06:59.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:06:59.706166 11883 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:00.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:06:59.707103 11884 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:00.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:00.894414 11886 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:00.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:00.897902 11885 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1189 milliseconds
14:07:00.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:00.898386 11877 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.192s user 0.359s sys 0.827s
14:07:00.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:00.898626 11877 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.192s user 0.359s sys 0.827s
14:07:00.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.898795 11877 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:00.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.899943 11877 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:00.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.902458 11877 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.903823 11877 hybrid_clock.cc:648] HybridClock initialized: now 1756994820903751 us; error 61 us; skew 500 ppm
14:07:00.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.904742 11877 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:00.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.911900 11877 webserver.cc:480] Webserver started at http://127.11.150.126:38503/ using document root <none> and password file <none>
14:07:00.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.912940 11877 fs_manager.cc:362] Metadata directory not provided
14:07:00.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.913187 11877 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:00.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.913820 11877 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:00.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.918089 11877 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/data/instance:
14:07:00.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ad7f368e15d4618b1ec458592806223"
14:07:00.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:00 on dist-test-slave-5mpg"
14:07:00.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.919282 11877 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/wal/instance:
14:07:00.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ad7f368e15d4618b1ec458592806223"
14:07:00.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:00 on dist-test-slave-5mpg"
14:07:00.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.927017 11877 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.005s
14:07:00.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.932758 11893 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:00.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.933934 11877 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.003s
14:07:00.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.934259 11877 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/wal
14:07:00.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6ad7f368e15d4618b1ec458592806223"
14:07:00.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:00 on dist-test-slave-5mpg"
14:07:00.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:00.934602 11877 fs_report.cc:389] FS layout report
14:07:00.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:00.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/wal
14:07:00.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/wal
14:07:00.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/data/data
14:07:00.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:00.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:00.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:00.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:00.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:00.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:00.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:00.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:00.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:00.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:00.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:01.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.019100 11877 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:01.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.020745 11877 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:01.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.021224 11877 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:01.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.091576 11877 rpc_server.cc:307] RPC server started. Bound to: 127.11.150.126:37365
14:07:01.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.091655 11944 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.150.126:37365 every 8 connection(s)
14:07:01.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.094352 11877 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/data/info.pb
14:07:01.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.099433 11945 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:01.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.101370 11865 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 11877
14:07:01.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.101804 11865 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-0/wal/instance
14:07:01.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.105293 11865 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:01.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:01.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/wal
14:07:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/data
14:07:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:01.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/logs
14:07:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/data/info.pb
14:07:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:01.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:01.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:01.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:01.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:01.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:01.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:01.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.125:37947
14:07:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.125
14:07:01.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:01.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:07:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:07:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:01.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41481/jwks.json with env {}
14:07:01.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.110440 11945 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } has no permanent_uuid. Determining permanent_uuid...
14:07:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.138752 11945 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } has no permanent_uuid. Determining permanent_uuid...
14:07:01.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.140851 11896 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.150.125:37947: connect: Connection refused (error 111)
14:07:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.144884 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.125:37947: Network error: Client connection negotiation failed: client connection to 127.11.150.125:37947: connect: Connection refused (error 111)
14:07:01.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.194317 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } attempt: 1
14:07:01.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.198248 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.125:37947: Network error: Client connection negotiation failed: client connection to 127.11.150.125:37947: connect: Connection refused (error 111)
14:07:01.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.266733 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } attempt: 2
14:07:01.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.270607 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.125:37947: Network error: Client connection negotiation failed: client connection to 127.11.150.125:37947: connect: Connection refused (error 111)
14:07:01.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.362064 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } attempt: 3
14:07:01.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.366176 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.125:37947: Network error: Client connection negotiation failed: client connection to 127.11.150.125:37947: connect: Connection refused (error 111)
14:07:01.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.403846 11947 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:01.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.404397 11947 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:01.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.405136 11947 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:01.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.436473 11947 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.436918 11947 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.437281 11947 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:01.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.437654 11947 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:01.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.438004 11947 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:01.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.438247 11947 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41481/jwks.json
14:07:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.473278 11947 master_runner.cc:386] Master server non-default flags:
14:07:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:07:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/data
14:07:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/wal
14:07:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:07:01.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.125:37947
14:07:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41481/jwks.json
14:07:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/data/info.pb
14:07:01.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.125
14:07:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:01.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:01.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/logs
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:01.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.475178 11947 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:01.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.477344 11947 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:01.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.489043 11956 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:01.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.509552 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } attempt: 4
14:07:01.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.513687 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.125:37947: Network error: Client connection negotiation failed: client connection to 127.11.150.125:37947: connect: Connection refused (error 111)
14:07:01.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:01.813124 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } attempt: 5
14:07:01.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.819581 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.125:37947: Network error: Client connection negotiation failed: client connection to 127.11.150.125:37947: connect: Connection refused (error 111)
14:07:02.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.367136 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } attempt: 6
14:07:02.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:02.386271 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.125:37947: Network error: Client connection negotiation failed: client connection to 127.11.150.125:37947: connect: Connection refused (error 111)
14:07:02.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:01.489774 11957 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:02.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:02.652383 11959 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:02.655220 11958 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1160 milliseconds
14:07:02.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.655294 11947 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:02.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.656507 11947 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:02.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.659651 11947 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:02.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.661056 11947 hybrid_clock.cc:648] HybridClock initialized: now 1756994822661013 us; error 50 us; skew 500 ppm
14:07:02.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.661840 11947 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:02.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.668121 11947 webserver.cc:480] Webserver started at http://127.11.150.125:43249/ using document root <none> and password file <none>
14:07:02.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.669080 11947 fs_manager.cc:362] Metadata directory not provided
14:07:02.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.669299 11947 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:02.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.669854 11947 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:02.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.673961 11947 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/data/instance:
14:07:02.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42d732147a7e4e219fc829e365123883"
14:07:02.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:02 on dist-test-slave-5mpg"
14:07:02.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.675047 11947 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/wal/instance:
14:07:02.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42d732147a7e4e219fc829e365123883"
14:07:02.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:02 on dist-test-slave-5mpg"
14:07:02.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.681732 11947 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
14:07:02.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.686969 11968 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:02.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.687927 11947 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:07:02.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.688254 11947 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/wal
14:07:02.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "42d732147a7e4e219fc829e365123883"
14:07:02.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:02 on dist-test-slave-5mpg"
14:07:02.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.688587 11947 fs_report.cc:389] FS layout report
14:07:02.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/wal
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/wal
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/data/data
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:02.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:02.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.777036 11947 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:02.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.778895 11947 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:02.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.779443 11947 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.852770 11947 rpc_server.cc:307] RPC server started. Bound to: 127.11.150.125:37947
14:07:02.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.852872 12019 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.150.125:37947 every 8 connection(s)
14:07:02.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.855623 11947 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/data/info.pb
14:07:02.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.860697 12020 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:02.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.865921 11865 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 11947
14:07:02.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.866341 11865 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-1/wal/instance
14:07:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.868500 11865 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:02.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/wal
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/data
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/logs
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/data/info.pb
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:02.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.124:34023
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.124
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:02.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:07:02.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:02.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41481/jwks.json with env {}
14:07:02.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.872195 12020 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } has no permanent_uuid. Determining permanent_uuid...
14:07:02.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.893985 12020 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } has no permanent_uuid. Determining permanent_uuid...
14:07:02.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.906457 12020 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } has no permanent_uuid. Determining permanent_uuid...
14:07:02.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:02.908810 11970 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:02.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:02.912595 12020 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:02.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:02.962023 12020 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 1
14:07:02.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:02.965755 12020 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:03.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.034137 12020 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 2
14:07:03.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.039664 12020 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:03.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.131178 12020 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 3
14:07:03.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.135233 12020 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:03.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.173173 12022 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:03.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.173650 12022 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:03.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.174396 12022 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.204416 12022 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:03.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.204682 12022 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:03.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.204874 12022 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:03.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.205073 12022 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:03.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.205327 12022 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:03.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.205480 12022 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41481/jwks.json
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.238011 12022 master_runner.cc:386] Master server non-default flags:
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/data
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/wal
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:03.238 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.124:34023
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41481/jwks.json
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/data/info.pb
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.124
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:03.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/logs
14:07:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:03.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.239259 12022 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:03.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.241097 12022 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:03.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.251698 12032 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:03.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.278596 12020 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 4
14:07:03.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.282080 12020 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:03.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.446786 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } attempt: 7
14:07:03.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.468079 11945 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } has no permanent_uuid. Determining permanent_uuid...
14:07:03.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.476521 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:03.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.535135 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 1
14:07:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.541375 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:03.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.581487 12020 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 5
14:07:03.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.587967 12020 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:03.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.622947 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 2
14:07:03.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.629083 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:03.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.714634 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 3
14:07:03.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.720813 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:03.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:03.861378 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 4
14:07:03.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.868206 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:04.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.135568 12020 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 6
14:07:04.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:04.145394 12020 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:04.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.151708 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 5
14:07:04.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:04.157573 11945 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.11.150.124:34023: Network error: Client connection negotiation failed: client connection to 127.11.150.124:34023: connect: Connection refused (error 111)
14:07:04.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.260767 12035 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:04.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:03.256544 12033 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:04.479211 12034 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1219 milliseconds
14:07:04.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.479331 12022 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:04.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.480737 12022 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:04.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.483916 12022 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:04.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.485536 12022 hybrid_clock.cc:648] HybridClock initialized: now 1756994824485498 us; error 30 us; skew 500 ppm
14:07:04.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.486583 12022 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:04.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.493701 12022 webserver.cc:480] Webserver started at http://127.11.150.124:43263/ using document root <none> and password file <none>
14:07:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.494905 12022 fs_manager.cc:362] Metadata directory not provided
14:07:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.495175 12022 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:04.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.495707 12022 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.502012 12022 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/data/instance:
14:07:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2831d5f25cc45709633433d3090f81f"
14:07:04.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:04 on dist-test-slave-5mpg"
14:07:04.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.503196 12022 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/wal/instance:
14:07:04.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2831d5f25cc45709633433d3090f81f"
14:07:04.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:04 on dist-test-slave-5mpg"
14:07:04.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.510113 12022 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.008s
14:07:04.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.515318 12047 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:04.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.516216 12022 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:07:04.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.516537 12022 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/wal
14:07:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c2831d5f25cc45709633433d3090f81f"
14:07:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:04 on dist-test-slave-5mpg"
14:07:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.516887 12022 fs_report.cc:389] FS layout report
14:07:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/wal
14:07:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/wal
14:07:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/data/data
14:07:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:04.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:04.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:04.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:04.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:04.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:04.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:04.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:04.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:04.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.578114 12022 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:04.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.579559 12022 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:04.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.580034 12022 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.650797 12022 rpc_server.cc:307] RPC server started. Bound to: 127.11.150.124:34023
14:07:04.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.650872 12098 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.150.124:34023 every 8 connection(s)
14:07:04.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.653513 12022 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/data/info.pb
14:07:04.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.658496 12099 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.662083 11865 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 12022
14:07:04.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.662541 11865 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/master-2/wal/instance
14:07:04.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.672981 12099 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } has no permanent_uuid. Determining permanent_uuid...
14:07:04.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.695741 12099 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } has no permanent_uuid. Determining permanent_uuid...
14:07:04.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.710103 11945 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 6
14:07:04.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.714306 12099 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } has no permanent_uuid. Determining permanent_uuid...
14:07:04.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.733369 11945 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223: Bootstrap starting.
14:07:04.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.734072 12099 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f: Bootstrap starting.
14:07:04.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.738809 11945 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223: Neither blocks nor log segments found. Creating new log.
14:07:04.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.740645 12099 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f: Neither blocks nor log segments found. Creating new log.
14:07:04.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.741429 11945 log.cc:826] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223: Log is configured to *not* fsync() on all Append() calls
14:07:04.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.742470 12099 log.cc:826] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f: Log is configured to *not* fsync() on all Append() calls
14:07:04.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.746603 11945 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223: No bootstrap required, opened a new log
14:07:04.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.749828 12099 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f: No bootstrap required, opened a new log
14:07:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.767551 11945 raft_consensus.cc:357] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } }
14:07:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.768160 11945 raft_consensus.cc:383] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:04.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.768348 11945 raft_consensus.cc:738] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6ad7f368e15d4618b1ec458592806223, State: Initialized, Role: FOLLOWER
14:07:04.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.769042 11945 consensus_queue.cc:260] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } }
14:07:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.769381 12099 raft_consensus.cc:357] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } }
14:07:04.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.770126 12099 raft_consensus.cc:383] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:04.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.770653 12099 raft_consensus.cc:738] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c2831d5f25cc45709633433d3090f81f, State: Initialized, Role: FOLLOWER
14:07:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.771679 12099 consensus_queue.cc:260] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } }
14:07:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.771787 12111 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } } }
14:07:04.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.772531 12111 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [sys.catalog]: This master's current role is: FOLLOWER
14:07:04.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.773347 11945 sys_catalog.cc:564] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [sys.catalog]: configured and running, proceeding with master startup.
14:07:04.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.775632 12112 sys_catalog.cc:455] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } } }
14:07:04.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.776618 12112 sys_catalog.cc:458] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [sys.catalog]: This master's current role is: FOLLOWER
14:07:04.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.777683 12099 sys_catalog.cc:564] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [sys.catalog]: configured and running, proceeding with master startup.
14:07:04.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.779309 12112 raft_consensus.cc:491] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:07:04.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.779788 12112 raft_consensus.cc:513] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } }
14:07:04.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:04.782271 11899 tablet.cc:2378] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:07:04.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.782886 12112 leader_election.cc:290] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6ad7f368e15d4618b1ec458592806223 (127.11.150.126:37365), 42d732147a7e4e219fc829e365123883 (127.11.150.125:37947)
14:07:04.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.783900 11920 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c2831d5f25cc45709633433d3090f81f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6ad7f368e15d4618b1ec458592806223" is_pre_election: true
14:07:04.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.785497 11920 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c2831d5f25cc45709633433d3090f81f in term 0.
14:07:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.787367 12050 leader_election.cc:304] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6ad7f368e15d4618b1ec458592806223, c2831d5f25cc45709633433d3090f81f; no voters:
14:07:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.788305 12112 raft_consensus.cc:2802] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [term 0 FOLLOWER]: Leader pre-election won for term 1
14:07:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.788650 12112 raft_consensus.cc:491] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:07:04.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.788980 12112 raft_consensus.cc:3058] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [term 0 FOLLOWER]: Advancing to term 1
14:07:04.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.797245 12112 raft_consensus.cc:513] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } }
14:07:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.800980 11920 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c2831d5f25cc45709633433d3090f81f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6ad7f368e15d4618b1ec458592806223"
14:07:04.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.801694 11920 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [term 0 FOLLOWER]: Advancing to term 1
14:07:04.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.804536 12112 leader_election.cc:290] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [CANDIDATE]: Term 1 election: Requested vote from peers 6ad7f368e15d4618b1ec458592806223 (127.11.150.126:37365), 42d732147a7e4e219fc829e365123883 (127.11.150.125:37947)
14:07:04.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.817083 11920 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c2831d5f25cc45709633433d3090f81f in term 1.
14:07:04.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.818259 12050 leader_election.cc:304] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6ad7f368e15d4618b1ec458592806223, c2831d5f25cc45709633433d3090f81f; no voters:
14:07:04.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.819042 12112 raft_consensus.cc:2802] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [term 1 FOLLOWER]: Leader election won for term 1
14:07:04.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.821844 12112 raft_consensus.cc:695] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [term 1 LEADER]: Becoming Leader. State: Replica: c2831d5f25cc45709633433d3090f81f, State: Running, Role: LEADER
14:07:04.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.822963 12112 consensus_queue.cc:237] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } }
14:07:04.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:04.835120 12053 tablet.cc:2378] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:07:04.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.849609 12133 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:07:04.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.860258 12125 sys_catalog.cc:455] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [sys.catalog]: SysCatalogTable state changed. Reason: New leader c2831d5f25cc45709633433d3090f81f. Latest consensus state: current_term: 1 leader_uuid: "c2831d5f25cc45709633433d3090f81f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } } }
14:07:04.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.861882 12125 sys_catalog.cc:458] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [sys.catalog]: This master's current role is: LEADER
14:07:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:04.885149 12136 catalog_manager.cc:1560] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:07:04.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:04.885478 12136 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:07:04.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.857278 11877 master_runner.cc:186] Error getting master registration for 127.11.150.125:37947: OK, instance_id { permanent_uuid: "42d732147a7e4e219fc829e365123883" instance_seqno: 1756994822832922 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:07:04.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.888585 11877 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:04.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.900772 12137 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:07:04.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.907076 12137 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:07:04.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.866822 12022 master_runner.cc:186] Error getting master registration for 127.11.150.125:37947: OK, instance_id { permanent_uuid: "42d732147a7e4e219fc829e365123883" instance_seqno: 1756994822832922 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:07:04.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.921490 12022 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:04.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.928941 11920 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [term 1 FOLLOWER]: Refusing update from remote peer c2831d5f25cc45709633433d3090f81f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:07:04.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.930495 12112 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [LEADER]: Connected to new peer: Peer: permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.947562 12111 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c2831d5f25cc45709633433d3090f81f. Latest consensus state: current_term: 1 leader_uuid: "c2831d5f25cc45709633433d3090f81f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } } }
14:07:04.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.948050 12111 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [sys.catalog]: This master's current role is: FOLLOWER
14:07:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.955843 12112 sys_catalog.cc:455] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c2831d5f25cc45709633433d3090f81f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } } }
14:07:04.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.956490 12112 sys_catalog.cc:458] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [sys.catalog]: This master's current role is: LEADER
14:07:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.959143 12137 catalog_manager.cc:1349] Generated new cluster ID: 3ce9f5dfafbb4e07adcd8be3dc973025
14:07:04.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.959396 12137 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:07:04.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.966984 12111 sys_catalog.cc:455] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c2831d5f25cc45709633433d3090f81f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } } }
14:07:04.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.967612 12111 sys_catalog.cc:458] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223 [sys.catalog]: This master's current role is: FOLLOWER
14:07:04.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.984961 12137 catalog_manager.cc:1372] Generated new certificate authority record
14:07:04.987 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:04.987284 12137 catalog_manager.cc:1506] Loading token signing keys...
14:07:05.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.010131 12137 catalog_manager.cc:5955] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f: Generated new TSK 0
14:07:05.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.011137 12137 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:07:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.046938 11865 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/wal
14:07:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/data
14:07:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:05.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/logs
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/data/info.pb
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.65:0
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.150.65
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.65
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:05.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:07:05.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.205843 12020 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } attempt: 7
14:07:05.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.228288 12020 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883: Bootstrap starting.
14:07:05.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.235239 12020 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883: Neither blocks nor log segments found. Creating new log.
14:07:05.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.237550 12020 log.cc:826] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883: Log is configured to *not* fsync() on all Append() calls
14:07:05.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.242740 12020 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883: No bootstrap required, opened a new log
14:07:05.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.267020 12020 raft_consensus.cc:357] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } }
14:07:05.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.268021 12020 raft_consensus.cc:383] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:05.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.268440 12020 raft_consensus.cc:738] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42d732147a7e4e219fc829e365123883, State: Initialized, Role: FOLLOWER
14:07:05.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.269395 12020 consensus_queue.cc:260] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } }
14:07:05.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.275135 12020 sys_catalog.cc:564] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [sys.catalog]: configured and running, proceeding with master startup.
14:07:05.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.275852 11993 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [term 0 FOLLOWER]: Advancing to term 1
14:07:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.276818 11995 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c2831d5f25cc45709633433d3090f81f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42d732147a7e4e219fc829e365123883" is_pre_election: true
14:07:05.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.278963 11994 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "c2831d5f25cc45709633433d3090f81f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42d732147a7e4e219fc829e365123883"
14:07:05.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.274268 12147 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } } }
14:07:05.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.282235 12147 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [sys.catalog]: This master's current role is: FOLLOWER
14:07:05.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.283473 11993 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [term 1 FOLLOWER]: Refusing update from remote peer c2831d5f25cc45709633433d3090f81f: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:07:05.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.285125 12125 consensus_queue.cc:1035] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [LEADER]: Connected to new peer: Peer: permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:05.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:05.289270 11974 tablet.cc:2378] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:07:05.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.338016 12147 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [sys.catalog]: SysCatalogTable state changed. Reason: New leader c2831d5f25cc45709633433d3090f81f. Latest consensus state: current_term: 1 leader_uuid: "c2831d5f25cc45709633433d3090f81f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } } }
14:07:05.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.339545 12147 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [sys.catalog]: This master's current role is: FOLLOWER
14:07:05.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.339121 12153 sys_catalog.cc:455] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "c2831d5f25cc45709633433d3090f81f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } } }
14:07:05.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.340792 12153 sys_catalog.cc:458] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883 [sys.catalog]: This master's current role is: FOLLOWER
14:07:05.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.381215 12163 catalog_manager.cc:1261] Loaded cluster ID: 3ce9f5dfafbb4e07adcd8be3dc973025
14:07:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.381738 12163 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883: loading cluster ID for follower catalog manager: success
14:07:05.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.389075 12163 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883: acquiring CA information for follower catalog manager: success
14:07:05.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.394030 12163 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 42d732147a7e4e219fc829e365123883: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:07:05.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:05.492352 12144 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:05.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:05.492976 12144 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:05.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:05.493625 12144 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:05.548709 12144 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:05.550207 12144 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.150.65
14:07:05.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.610532 12144 tablet_server_runner.cc:78] Tablet server non-default flags:
14:07:05.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:05.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:07:05.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:05.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/data
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/wal
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.65:0
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/data/info.pb
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.65
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.150.65
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/logs
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:05.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:07:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:05.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.612239 12144 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:05.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.614178 12144 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:05.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:05.629870 12170 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:05.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.695946 12125 sys_catalog.cc:455] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "c2831d5f25cc45709633433d3090f81f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6ad7f368e15d4618b1ec458592806223" member_type: VOTER last_known_addr { host: "127.11.150.126" port: 37365 } } peers { permanent_uuid: "42d732147a7e4e219fc829e365123883" member_type: VOTER last_known_addr { host: "127.11.150.125" port: 37947 } } peers { permanent_uuid: "c2831d5f25cc45709633433d3090f81f" member_type: VOTER last_known_addr { host: "127.11.150.124" port: 34023 } } }
14:07:05.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.696723 12125 sys_catalog.cc:458] T 00000000000000000000000000000000 P c2831d5f25cc45709633433d3090f81f [sys.catalog]: This master's current role is: LEADER
14:07:05.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.873629 12133 catalog_manager.cc:1261] Loaded cluster ID: 3ce9f5dfafbb4e07adcd8be3dc973025
14:07:05.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.874528 12133 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223: loading cluster ID for follower catalog manager: success
14:07:05.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.888784 12133 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223: acquiring CA information for follower catalog manager: success
14:07:05.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:05.899655 12133 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 6ad7f368e15d4618b1ec458592806223: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:07:08.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:05.630162 12171 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:08.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:08.800540 12172 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3169 milliseconds
14:07:08.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:07.030349 12169 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 12144
14:07:08.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:08.808610 12144 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.177s user 0.000s sys 0.006s
14:07:08.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:08.809562 12144 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.178s user 0.001s sys 0.006s
14:07:08.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.810951 12144 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:08.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:08.811728 12174 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:08.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.814158 12144 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:08.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.820983 12144 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:08.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.823027 12144 hybrid_clock.cc:648] HybridClock initialized: now 1756994828822952 us; error 71 us; skew 500 ppm
14:07:08.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.824190 12144 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:08.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.855233 12144 webserver.cc:480] Webserver started at http://127.11.150.65:41825/ using document root <none> and password file <none>
14:07:08.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.856657 12144 fs_manager.cc:362] Metadata directory not provided
14:07:08.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.857093 12144 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:08.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.857718 12144 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:08.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.863546 12144 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/data/instance:
14:07:08.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6071d5bace20452d931bfdde949866d2"
14:07:08.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:08 on dist-test-slave-5mpg"
14:07:08.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.865105 12144 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/wal/instance:
14:07:08.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6071d5bace20452d931bfdde949866d2"
14:07:08.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:08 on dist-test-slave-5mpg"
14:07:08.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.883468 12144 fs_manager.cc:696] Time spent creating directory manager: real 0.015s user 0.004s sys 0.007s
14:07:08.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.897796 12184 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.899063 12144 fs_manager.cc:730] Time spent opening block manager: real 0.009s user 0.004s sys 0.004s
14:07:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.899405 12144 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/wal
14:07:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6071d5bace20452d931bfdde949866d2"
14:07:08.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:08 on dist-test-slave-5mpg"
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.899804 12144 fs_report.cc:389] FS layout report
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/wal
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/wal
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/data/data
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:08.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:08.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:08.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:08.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:08.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:08.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.955936 12144 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:08.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.957808 12144 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:08.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.958523 12144 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:08.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.961565 12144 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:07:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.966209 12144 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:07:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.966405 12144 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:07:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.966693 12144 ts_tablet_manager.cc:610] Registered 0 tablets
14:07:08.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:08.966933 12144 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:07:09.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.197307 12144 rpc_server.cc:307] RPC server started. Bound to: 127.11.150.65:44057
14:07:09.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.197404 12296 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.150.65:44057 every 8 connection(s)
14:07:09.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.200220 12144 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/data/info.pb
14:07:09.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.204926 11865 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 12144
14:07:09.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.205468 11865 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-0/wal/instance
14:07:09.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.264221 11865 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:09.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:09.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/wal
14:07:09.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/data
14:07:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/logs
14:07:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/data/info.pb
14:07:09.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.66:0
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.150.66
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.66
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:09.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:07:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.292860 12297 heartbeater.cc:344] Connected to a master server at 127.11.150.124:34023
14:07:09.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.293505 12297 heartbeater.cc:461] Registering TS with master...
14:07:09.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.295470 12297 heartbeater.cc:507] Master 127.11.150.124:34023 requested a full tablet report, sending...
14:07:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.300391 12064 ts_manager.cc:194] Registered new tserver with Master: 6071d5bace20452d931bfdde949866d2 (127.11.150.65:44057)
14:07:09.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.303737 12064 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.150.65:44553
14:07:09.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.343608 12298 heartbeater.cc:344] Connected to a master server at 127.11.150.125:37947
14:07:09.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.344002 12298 heartbeater.cc:461] Registering TS with master...
14:07:09.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.344655 12298 heartbeater.cc:507] Master 127.11.150.125:37947 requested a full tablet report, sending...
14:07:09.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.348783 11985 ts_manager.cc:194] Registered new tserver with Master: 6071d5bace20452d931bfdde949866d2 (127.11.150.65:44057)
14:07:09.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.362035 12302 heartbeater.cc:344] Connected to a master server at 127.11.150.126:37365
14:07:09.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.362358 12302 heartbeater.cc:461] Registering TS with master...
14:07:09.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.362911 12302 heartbeater.cc:507] Master 127.11.150.126:37365 requested a full tablet report, sending...
14:07:09.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.365428 11910 ts_manager.cc:194] Registered new tserver with Master: 6071d5bace20452d931bfdde949866d2 (127.11.150.65:44057)
14:07:09.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:09.643330 12305 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:09.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:09.643934 12305 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:09.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:09.644501 12305 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:09.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:09.674929 12305 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:09.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:09.675868 12305 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.150.66
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.709858 12305 tablet_server_runner.cc:78] Tablet server non-default flags:
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/data
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/wal
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.66:0
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:09.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/data/info.pb
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.66
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.150.66
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/logs
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:09.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:09.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.711462 12305 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:09.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:09.713464 12305 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:09.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:09.727648 12315 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:10.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:10.314357 12297 heartbeater.cc:499] Master 127.11.150.124:34023 was elected leader, sending a full tablet report...
14:07:11.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.128850 12314 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 12305
14:07:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.271772 12314 kernel_stack_watchdog.cc:198] Thread 12305 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 397ms:
14:07:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:07:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:07:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:07:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:07:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.271947 12305 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.543s user 0.527s sys 0.959s
14:07:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.272317 12305 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.543s user 0.527s sys 0.959s
14:07:11.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:09.728910 12316 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:11.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.274088 12318 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.276232 12317 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1547 milliseconds
14:07:11.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.276285 12305 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:11.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.277478 12305 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:11.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.279379 12305 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:11.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.280687 12305 hybrid_clock.cc:648] HybridClock initialized: now 1756994831280639 us; error 39 us; skew 500 ppm
14:07:11.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.281524 12305 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:11.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.287163 12305 webserver.cc:480] Webserver started at http://127.11.150.66:38983/ using document root <none> and password file <none>
14:07:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.288005 12305 fs_manager.cc:362] Metadata directory not provided
14:07:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.288216 12305 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:11.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.288728 12305 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.292945 12305 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/data/instance:
14:07:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9fc1310d78f4dcd985ae9af85b94d62"
14:07:11.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:11 on dist-test-slave-5mpg"
14:07:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.294040 12305 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/wal/instance:
14:07:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9fc1310d78f4dcd985ae9af85b94d62"
14:07:11.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:11 on dist-test-slave-5mpg"
14:07:11.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.300894 12305 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.002s
14:07:11.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.306473 12326 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.307332 12305 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
14:07:11.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.307657 12305 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/wal
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a9fc1310d78f4dcd985ae9af85b94d62"
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:11 on dist-test-slave-5mpg"
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.307932 12305 fs_report.cc:389] FS layout report
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/wal
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/wal
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/data/data
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:11.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:11.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:11.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:11.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:11.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:11.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.361716 12305 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.362977 12305 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:11.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.363390 12305 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:11.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.365592 12305 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:07:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.369240 12305 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:07:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.369434 12305 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:07:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.369725 12305 ts_tablet_manager.cc:610] Registered 0 tablets
14:07:11.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.369920 12305 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:07:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.502862 12305 rpc_server.cc:307] RPC server started. Bound to: 127.11.150.66:39329
14:07:11.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.502913 12439 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.150.66:39329 every 8 connection(s)
14:07:11.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.505579 12305 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/data/info.pb
14:07:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.514140 11865 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 12305
14:07:11.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.514613 11865 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-1/wal/instance
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.526773 11865 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/wal
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/data
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/logs
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/data/info.pb
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:11.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:07:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:11.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.67:0
14:07:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.150.67
14:07:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.67
14:07:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:07:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:07:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:11.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:07:11.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.545512 12440 heartbeater.cc:344] Connected to a master server at 127.11.150.124:34023
14:07:11.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.545977 12440 heartbeater.cc:461] Registering TS with master...
14:07:11.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.547142 12440 heartbeater.cc:507] Master 127.11.150.124:34023 requested a full tablet report, sending...
14:07:11.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.549521 12064 ts_manager.cc:194] Registered new tserver with Master: a9fc1310d78f4dcd985ae9af85b94d62 (127.11.150.66:39329)
14:07:11.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.550627 12441 heartbeater.cc:344] Connected to a master server at 127.11.150.125:37947
14:07:11.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.550959 12441 heartbeater.cc:461] Registering TS with master...
14:07:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.551313 12064 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.150.66:51603
14:07:11.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.551846 12441 heartbeater.cc:507] Master 127.11.150.125:37947 requested a full tablet report, sending...
14:07:11.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.554297 11985 ts_manager.cc:194] Registered new tserver with Master: a9fc1310d78f4dcd985ae9af85b94d62 (127.11.150.66:39329)
14:07:11.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.556998 12442 heartbeater.cc:344] Connected to a master server at 127.11.150.126:37365
14:07:11.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.557256 12442 heartbeater.cc:461] Registering TS with master...
14:07:11.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.557721 12442 heartbeater.cc:507] Master 127.11.150.126:37365 requested a full tablet report, sending...
14:07:11.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.559666 11910 ts_manager.cc:194] Registered new tserver with Master: a9fc1310d78f4dcd985ae9af85b94d62 (127.11.150.66:39329)
14:07:11.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.816871 12450 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:11.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.817312 12450 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:11.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.817847 12450 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:11.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.846706 12450 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:11.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.847536 12450 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.11.150.67
14:07:11.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.885540 12450 tablet_server_runner.cc:78] Tablet server non-default flags:
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.11.150.84:46603
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/data
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/wal
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.11.150.67:0
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/data/info.pb
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.11.150.67
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.11.150.126:37365,127.11.150.125:37947,127.11.150.124:34023
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.11.150.67
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/logs
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:11.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:11.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:11.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:07:11.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:11.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:11.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:11.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:11.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:11.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:11.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.887091 12450 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:11.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:11.888937 12450 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:11.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.901154 12457 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:12.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:12.138276 12095 debug-util.cc:398] Leaking SignalData structure 0x7b080006f400 after lost signal to thread 12027
14:07:12.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:12.555320 12440 heartbeater.cc:499] Master 127.11.150.124:34023 was elected leader, sending a full tablet report...
14:07:13.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.904932 12460 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:13.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:11.903196 12458 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:13.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:13.065462 12459 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection time-out
14:07:13.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.065521 12450 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:13.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.070758 12450 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:13.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.073642 12450 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:13.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.075222 12450 hybrid_clock.cc:648] HybridClock initialized: now 1756994833075183 us; error 39 us; skew 500 ppm
14:07:13.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.076383 12450 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:13.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.083961 12450 webserver.cc:480] Webserver started at http://127.11.150.67:43295/ using document root <none> and password file <none>
14:07:13.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.085245 12450 fs_manager.cc:362] Metadata directory not provided
14:07:13.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.085520 12450 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:13.086 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.086102 12450 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:13.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.093214 12450 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/data/instance:
14:07:13.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "97a62398b01b48dbb5d424dfd09a1469"
14:07:13.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:13 on dist-test-slave-5mpg"
14:07:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.094769 12450 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/wal/instance:
14:07:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "97a62398b01b48dbb5d424dfd09a1469"
14:07:13.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:13 on dist-test-slave-5mpg"
14:07:13.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.105086 12450 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.010s sys 0.000s
14:07:13.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.112913 12468 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:13.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.114138 12450 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
14:07:13.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.114523 12450 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/wal
14:07:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "97a62398b01b48dbb5d424dfd09a1469"
14:07:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:13 on dist-test-slave-5mpg"
14:07:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.114943 12450 fs_report.cc:389] FS layout report
14:07:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/wal
14:07:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/wal
14:07:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/data/data
14:07:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:13.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:13.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:13.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.180667 12450 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:13.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.182650 12450 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:13.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.183210 12450 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:13.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.186631 12450 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:07:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.193028 12450 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:07:13.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.193316 12450 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:07:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.193734 12450 ts_tablet_manager.cc:610] Registered 0 tablets
14:07:13.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.193982 12450 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:07:13.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.347556 12450 rpc_server.cc:307] RPC server started. Bound to: 127.11.150.67:45407
14:07:13.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.347607 12581 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.11.150.67:45407 every 8 connection(s)
14:07:13.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.351140 12450 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/data/info.pb
14:07:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.355736 11865 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 12450
14:07:13.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.356176 11865 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster313384471436060671/ts-2/wal/instance
14:07:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.419706 12582 heartbeater.cc:344] Connected to a master server at 127.11.150.124:34023
14:07:13.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.420193 12582 heartbeater.cc:461] Registering TS with master...
14:07:13.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.421517 12582 heartbeater.cc:507] Master 127.11.150.124:34023 requested a full tablet report, sending...
14:07:13.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.421808 12583 heartbeater.cc:344] Connected to a master server at 127.11.150.125:37947
14:07:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.422112 12583 heartbeater.cc:461] Registering TS with master...
14:07:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.422866 12583 heartbeater.cc:507] Master 127.11.150.125:37947 requested a full tablet report, sending...
14:07:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.424057 12584 heartbeater.cc:344] Connected to a master server at 127.11.150.126:37365
14:07:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.424360 12584 heartbeater.cc:461] Registering TS with master...
14:07:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.425160 12584 heartbeater.cc:507] Master 127.11.150.126:37365 requested a full tablet report, sending...
14:07:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.425712 11985 ts_manager.cc:194] Registered new tserver with Master: 97a62398b01b48dbb5d424dfd09a1469 (127.11.150.67:45407)
14:07:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.427160 12064 ts_manager.cc:194] Registered new tserver with Master: 97a62398b01b48dbb5d424dfd09a1469 (127.11.150.67:45407)
14:07:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.427683 11909 ts_manager.cc:194] Registered new tserver with Master: 97a62398b01b48dbb5d424dfd09a1469 (127.11.150.67:45407)
14:07:13.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.428932 12064 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.11.150.67:48175
14:07:13.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:13.447697 11865 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:07:13.457 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:07:13.461 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:07:13.476 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.11.150.126"
port: 37365
}
bound_http_address {
host: "127.11.150.126"
port: 38503
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.11.150.125"
port: 37947
}
bound_http_address {
host: "127.11.150.125"
port: 43249
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.11.150.124"
port: 34023
}
bound_http_address {
host: "127.11.150.124"
port: 43263
}
}
}
14:07:13.481 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:07:13.484 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.11.150.65"
port: 44057
}
bound_http_address {
host: "127.11.150.65"
port: 41825
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.11.150.66"
port: 39329
}
bound_http_address {
host: "127.11.150.66"
port: 38983
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.11.150.67"
port: 45407
}
bound_http_address {
host: "127.11.150.67"
port: 43295
}
}
}
14:07:13.485 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:07:13.686 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
14:07:13.722 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:07:13.875 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskKowyQv/test-tmp
14:07:13.877 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:07:13.877 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskKowyQv/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:07:13.895 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:07:14.237 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:14.237 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:07:14.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:14.240540 12593 mini_oidc.cc:150] Starting JWKS server
14:07:14.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:14.243747 12593 webserver.cc:480] Webserver started at http://127.0.0.1:38081/ using document root <none> and password file <none>
14:07:14.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:14.244130 12593 mini_oidc.cc:186] Starting OIDC Discovery server
14:07:14.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:14.245641 12593 webserver.cc:480] Webserver started at http://127.0.0.1:41525/ using document root <none> and password file <none>
14:07:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:07:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/principal' for realm 'KRBTEST.COM',
14:07:14.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:07:14.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:14 dist-test-slave-5mpg krb5kdc[12605](info): setting up network...
14:07:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:07:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:14 dist-test-slave-5mpg krb5kdc[12605](info): set up 2 sockets
14:07:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:14 dist-test-slave-5mpg krb5kdc[12605](info): commencing operation
14:07:14.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:07:14.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:14.432171 12582 heartbeater.cc:499] Master 127.11.150.124:34023 was elected leader, sending a full tablet report...
14:07:16.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.255124 12593 mini_kdc.cc:121] Time spent starting KDC: real 1.976s user 0.005s sys 0.012s
14:07:16.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:07:16.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:07:16.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:07:16.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:07:16.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:07:16.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:07:16.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:07:16.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:07:16.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:07:16.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:07:16.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/test-user.keytab.
14:07:16.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/test-user.keytab.
14:07:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:16 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994836, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:16.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:07:16.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T14:07:16Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:07:16.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T14:07:16Z Disabled control of system clock
14:07:16.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.76.126@KRBTEST.COM; defaulting to no policy
14:07:16.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:16.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.76.126@KRBTEST.COM" created.
14:07:16.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:16.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:16.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:16.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.76.126@KRBTEST.COM; defaulting to no policy
14:07:16.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:16.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.76.126@KRBTEST.COM" created.
14:07:16.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:16.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.126 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:16.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.126 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:16.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:16.462339 12593 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:16.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:16.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal
14:07:16.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data
14:07:16.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:16.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:16.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/logs
14:07:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data/info.pb
14:07:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:16.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.126:37999
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.126
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.126
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:16.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:16.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:16.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:07:16.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:07:16.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.751816 12624 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:16.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.752311 12624 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:16.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.752924 12624 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.783272 12624 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:07:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.783490 12624 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:16.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.783777 12624 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.783982 12624 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.784235 12624 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:16.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.784497 12624 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:16.785 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.784788 12624 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38081/jwks.json
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:16.819573 12624 master_runner.cc:386] Master server non-default flags:
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.126
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.126:37999
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:07:16.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:07:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data/info.pb
14:07:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.126
14:07:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:16.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/logs
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:16.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:16.821224 12624 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:16.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:16.822602 12624 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:16.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.833460 12631 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:18.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:16.834333 12632 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:18.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.051311 12634 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:18.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.053597 12633 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1216 milliseconds
14:07:18.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.053687 12624 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:18.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.054836 12624 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:18.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.057112 12624 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:18.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.058405 12624 hybrid_clock.cc:648] HybridClock initialized: now 1756994838058359 us; error 49 us; skew 500 ppm
14:07:18.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.059092 12624 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:18.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:18 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994838, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:18.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.064821 12624 init.cc:377] Logged in from keytab as kudu/127.12.76.126@KRBTEST.COM (short username kudu)
14:07:18.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.069749 12624 webserver.cc:480] Webserver started at http://127.12.76.126:43093/ using document root <none> and password file <none>
14:07:18.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.070596 12624 fs_manager.cc:362] Metadata directory not provided
14:07:18.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.070847 12624 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:18.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.071257 12624 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:18.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.075301 12624 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data/instance:
14:07:18.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33e3a480d48e42138371de8fb92f5cec"
14:07:18.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:18 on dist-test-slave-5mpg"
14:07:18.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.076335 12624 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal/instance:
14:07:18.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33e3a480d48e42138371de8fb92f5cec"
14:07:18.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:18 on dist-test-slave-5mpg"
14:07:18.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.082967 12624 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
14:07:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.087824 12644 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:18.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.088726 12624 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:07:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.089011 12624 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal
14:07:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33e3a480d48e42138371de8fb92f5cec"
14:07:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:18 on dist-test-slave-5mpg"
14:07:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.089251 12624 fs_report.cc:389] FS layout report
14:07:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal
14:07:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal
14:07:18.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data/data
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:18.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:18.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.156831 12624 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:18.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.162443 12624 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:18.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.162945 12624 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:18.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.236554 12624 rpc_server.cc:307] RPC server started. Bound to: 127.12.76.126:37999
14:07:18.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.236683 12695 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.76.126:37999 every 8 connection(s)
14:07:18.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.239431 12624 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data/info.pb
14:07:18.244 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.244551 12696 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.249817 12593 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 12624
14:07:18.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.250227 12593 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal/instance
14:07:18.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.254819 12696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } has no permanent_uuid. Determining permanent_uuid...
14:07:18.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.76.125@KRBTEST.COM; defaulting to no policy
14:07:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:18.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.76.125@KRBTEST.COM" created.
14:07:18.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:18 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994838, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.126@KRBTEST.COM for kudu/127.12.76.126@KRBTEST.COM
14:07:18.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:18.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:18.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:18.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.290838 12696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } has no permanent_uuid. Determining permanent_uuid...
14:07:18.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.293314 12646 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.76.125:37303: connect: Connection refused (error 111)
14:07:18.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.297297 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.125:37303: Network error: Client connection negotiation failed: client connection to 127.12.76.125:37303: connect: Connection refused (error 111)
14:07:18.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.76.125@KRBTEST.COM; defaulting to no policy
14:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:18.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.76.125@KRBTEST.COM" created.
14:07:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.125 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:18.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.125 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.328663 12593 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal
14:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data
14:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:18.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/logs
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data/info.pb
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.125:37303
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.125
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.125
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:07:18.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:07:18.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.346774 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } attempt: 1
14:07:18.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.351661 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.125:37303: Network error: Client connection negotiation failed: client connection to 127.12.76.125:37303: connect: Connection refused (error 111)
14:07:18.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.420095 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } attempt: 2
14:07:18.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.423789 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.125:37303: Network error: Client connection negotiation failed: client connection to 127.12.76.125:37303: connect: Connection refused (error 111)
14:07:18.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.515273 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } attempt: 3
14:07:18.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.519451 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.125:37303: Network error: Client connection negotiation failed: client connection to 127.12.76.125:37303: connect: Connection refused (error 111)
14:07:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.642082 12705 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:18.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.642587 12705 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:18.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.643194 12705 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:18.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.662827 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } attempt: 4
14:07:18.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.666826 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.125:37303: Network error: Client connection negotiation failed: client connection to 127.12.76.125:37303: connect: Connection refused (error 111)
14:07:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.675858 12705 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:07:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.676065 12705 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.676321 12705 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.676510 12705 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:18.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.676761 12705 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:18.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.676950 12705 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:18.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.677089 12705 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38081/jwks.json
14:07:18.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.713001 12705 master_runner.cc:386] Master server non-default flags:
14:07:18.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:18.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.125
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.125:37303
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data/info.pb
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.125
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:18.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/logs
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.714190 12705 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:18.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.715309 12705 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:18.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.725791 12711 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:18.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:18.966229 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } attempt: 5
14:07:18.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.972509 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.125:37303: Network error: Client connection negotiation failed: client connection to 127.12.76.125:37303: connect: Connection refused (error 111)
14:07:19.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.520066 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } attempt: 6
14:07:19.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:19.532727 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.125:37303: Network error: Client connection negotiation failed: client connection to 127.12.76.125:37303: connect: Connection refused (error 111)
14:07:19.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:18.726390 12712 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:19.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:19.961602 12714 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:19.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:19.966281 12713 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1237 milliseconds
14:07:19.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.966380 12705 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:19.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.967607 12705 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.970499 12705 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:19.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.971942 12705 hybrid_clock.cc:648] HybridClock initialized: now 1756994839971897 us; error 54 us; skew 500 ppm
14:07:19.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.972797 12705 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:19.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:19 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994839, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:19.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.978904 12705 init.cc:377] Logged in from keytab as kudu/127.12.76.125@KRBTEST.COM (short username kudu)
14:07:19.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.984341 12705 webserver.cc:480] Webserver started at http://127.12.76.125:42335/ using document root <none> and password file <none>
14:07:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.985319 12705 fs_manager.cc:362] Metadata directory not provided
14:07:19.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.985543 12705 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:19.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.986063 12705 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:19.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.990397 12705 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data/instance:
14:07:19.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8476c8ea00b544c8b5f09adf04b7539a"
14:07:19.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:19 on dist-test-slave-5mpg"
14:07:19.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.991525 12705 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal/instance:
14:07:19.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8476c8ea00b544c8b5f09adf04b7539a"
14:07:19.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:19 on dist-test-slave-5mpg"
14:07:19.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:19.998474 12705 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.001s sys 0.008s
14:07:20.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.003798 12728 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:20.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.004741 12705 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
14:07:20.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.005087 12705 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal
14:07:20.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8476c8ea00b544c8b5f09adf04b7539a"
14:07:20.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:19 on dist-test-slave-5mpg"
14:07:20.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.005404 12705 fs_report.cc:389] FS layout report
14:07:20.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data/data
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:20.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:20.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.060276 12705 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.069816 12705 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:20.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.070261 12705 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:20.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.146428 12705 rpc_server.cc:307] RPC server started. Bound to: 127.12.76.125:37303
14:07:20.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.146503 12779 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.76.125:37303 every 8 connection(s)
14:07:20.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.149039 12705 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data/info.pb
14:07:20.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.149907 12593 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 12705
14:07:20.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.150362 12593 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal/instance
14:07:20.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.155061 12780 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:20.167 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.76.124@KRBTEST.COM; defaulting to no policy
14:07:20.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:20.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.76.124@KRBTEST.COM" created.
14:07:20.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.170740 12780 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } has no permanent_uuid. Determining permanent_uuid...
14:07:20.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:20 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994839, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.125@KRBTEST.COM for kudu/127.12.76.126@KRBTEST.COM
14:07:20.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:20.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:20.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:20.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.202908 12780 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } has no permanent_uuid. Determining permanent_uuid...
14:07:20.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.76.124@KRBTEST.COM; defaulting to no policy
14:07:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:20.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.76.124@KRBTEST.COM" created.
14:07:20.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:20 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994839, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.125@KRBTEST.COM for kudu/127.12.76.125@KRBTEST.COM
14:07:20.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.228726 12780 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } has no permanent_uuid. Determining permanent_uuid...
14:07:20.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:20.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.124 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:20.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.124 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:20.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.230834 12730 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:20.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.232383 12593 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:20.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:20.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal
14:07:20.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data
14:07:20.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:20.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/logs
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data/info.pb
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.124:44285
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.124
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.124
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:20.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:20.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:20.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:20.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:20.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:20.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:20.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:07:20.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:07:20.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.234922 12780 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:20.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.284348 12780 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 1
14:07:20.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.288332 12780 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:20.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.356652 12780 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 2
14:07:20.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.361402 12780 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:20.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.452822 12780 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 3
14:07:20.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.458163 12780 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.551757 12790 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:20.552 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.552250 12790 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:20.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.552907 12790 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.582903 12790 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:07:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.583106 12790 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.583341 12790 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.583537 12790 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:20.583 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.583750 12790 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.583952 12790 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:20.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.584102 12790 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38081/jwks.json
14:07:20.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.593417 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } attempt: 7
14:07:20.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.601544 12780 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 4
14:07:20.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:20 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994838, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.126@KRBTEST.COM for kudu/127.12.76.125@KRBTEST.COM
14:07:20.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.605679 12780 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:20.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.614326 12696 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } has no permanent_uuid. Determining permanent_uuid...
14:07:20.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.619751 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:20.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.627624 12790 master_runner.cc:386] Master server non-default flags:
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.124
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.124:44285
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:20.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data/info.pb
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.124
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/logs
14:07:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.629246 12790 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:20.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.630550 12790 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:20.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.641548 12798 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:20.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.678754 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 1
14:07:20.686 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.686305 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:20.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.767823 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 2
14:07:20.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.774189 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:20.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.859720 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 3
14:07:20.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.864671 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:20.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:20.905063 12780 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 5
14:07:20.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.909551 12780 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:21.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:21.005098 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 4
14:07:21.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:21.011041 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:21.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:21.294551 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 5
14:07:21.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:21.300577 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:21.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:21.457065 12780 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 6
14:07:21.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:21.477746 12780 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:21.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:21.695797 12293 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac720 after lost signal to thread 12165
14:07:21.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:21.697485 12293 debug-util.cc:398] Leaking SignalData structure 0x7b08000b2f60 after lost signal to thread 12296
14:07:21.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:21.853103 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 6
14:07:21.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:21.895741 12696 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.12.76.124:44285: Network error: Client connection negotiation failed: client connection to 127.12.76.124:44285: connect: Connection refused (error 111)
14:07:22.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:20.642521 12799 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:22.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:22.071902 12801 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:22.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:22.075958 12800 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1430 milliseconds
14:07:22.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.076113 12790 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:22.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.077870 12790 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:22.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.081013 12790 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:22.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.082616 12790 hybrid_clock.cc:648] HybridClock initialized: now 1756994842082545 us; error 71 us; skew 500 ppm
14:07:22.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.083740 12790 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:22.089 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:22 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994842, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:22.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.090905 12790 init.cc:377] Logged in from keytab as kudu/127.12.76.124@KRBTEST.COM (short username kudu)
14:07:22.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.100713 12790 webserver.cc:480] Webserver started at http://127.12.76.124:37105/ using document root <none> and password file <none>
14:07:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.101573 12790 fs_manager.cc:362] Metadata directory not provided
14:07:22.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.101791 12790 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:22.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.102278 12790 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:22.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.106420 12790 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data/instance:
14:07:22.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "122166f9788348b3b2e7f7f0b21d6ae7"
14:07:22.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:22 on dist-test-slave-5mpg"
14:07:22.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.107674 12790 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal/instance:
14:07:22.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "122166f9788348b3b2e7f7f0b21d6ae7"
14:07:22.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:22 on dist-test-slave-5mpg"
14:07:22.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.114964 12790 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.003s
14:07:22.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.121644 12815 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:22.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.122912 12790 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.000s sys 0.005s
14:07:22.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.123407 12790 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal
14:07:22.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "122166f9788348b3b2e7f7f0b21d6ae7"
14:07:22.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:22 on dist-test-slave-5mpg"
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.123724 12790 fs_report.cc:389] FS layout report
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data/data
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:22.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:22.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.177765 12790 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:22.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.183065 12790 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:22.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.183561 12790 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:22.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.273684 12790 rpc_server.cc:307] RPC server started. Bound to: 127.12.76.124:44285
14:07:22.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.273746 12866 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.76.124:44285 every 8 connection(s)
14:07:22.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.276466 12790 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data/info.pb
14:07:22.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.277839 12593 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 12790
14:07:22.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.278373 12593 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal/instance
14:07:22.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.282706 12867 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:07:22.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.298723 12867 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } has no permanent_uuid. Determining permanent_uuid...
14:07:22.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:22 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994842, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.124@KRBTEST.COM for kudu/127.12.76.126@KRBTEST.COM
14:07:22.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.356890 12867 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } has no permanent_uuid. Determining permanent_uuid...
14:07:22.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:22 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994836, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.76.126@KRBTEST.COM
14:07:22.373 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:22 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994842, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.124@KRBTEST.COM for kudu/127.12.76.125@KRBTEST.COM
14:07:22.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.387706 12867 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } has no permanent_uuid. Determining permanent_uuid...
14:07:22.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:22 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994836, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.76.125@KRBTEST.COM
14:07:22.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:22 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994842, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.124@KRBTEST.COM for kudu/127.12.76.124@KRBTEST.COM
14:07:22.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:22 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994836, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.12.76.124@KRBTEST.COM
14:07:22.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.424065 12867 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: Bootstrap starting.
14:07:22.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.431419 12867 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: Neither blocks nor log segments found. Creating new log.
14:07:22.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.433198 12867 log.cc:826] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: Log is configured to *not* fsync() on all Append() calls
14:07:22.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.438326 12867 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: No bootstrap required, opened a new log
14:07:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.457957 12867 raft_consensus.cc:357] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:22.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.458599 12867 raft_consensus.cc:383] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:22.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.458880 12867 raft_consensus.cc:738] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 122166f9788348b3b2e7f7f0b21d6ae7, State: Initialized, Role: FOLLOWER
14:07:22.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.459689 12867 consensus_queue.cc:260] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.463709 12878 sys_catalog.cc:455] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:22.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.465520 12878 sys_catalog.cc:458] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: This master's current role is: FOLLOWER
14:07:22.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.465828 12867 sys_catalog.cc:564] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: configured and running, proceeding with master startup.
14:07:22.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.472281 12878 raft_consensus.cc:491] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:07:22.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.472781 12878 raft_consensus.cc:513] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:22.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.475086 12878 leader_election.cc:290] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 33e3a480d48e42138371de8fb92f5cec (127.12.76.126:37999), 8476c8ea00b544c8b5f09adf04b7539a (127.12.76.125:37303)
14:07:22.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:22.531064 12889 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:07:22.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:22.531668 12889 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:07:22.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.506147 12790 master_runner.cc:186] Error getting master registration for 127.12.76.126:37999: OK, instance_id { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" instance_seqno: 1756994838217224 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:07:22.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.537068 12790 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:22.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.538203 12780 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 7
14:07:22.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:22 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994839, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.125@KRBTEST.COM for kudu/127.12.76.124@KRBTEST.COM
14:07:22.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.576371 12780 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: Bootstrap starting.
14:07:22.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.586009 12780 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: Neither blocks nor log segments found. Creating new log.
14:07:22.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.589175 12780 log.cc:826] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: Log is configured to *not* fsync() on all Append() calls
14:07:22.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.596309 12780 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: No bootstrap required, opened a new log
14:07:22.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.629540 12780 raft_consensus.cc:357] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:22.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.630595 12780 raft_consensus.cc:383] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:22.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.631168 12780 raft_consensus.cc:738] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8476c8ea00b544c8b5f09adf04b7539a, State: Initialized, Role: FOLLOWER
14:07:22.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.632457 12780 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:22.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.639151 12780 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: configured and running, proceeding with master startup.
14:07:22.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.641249 12755 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8476c8ea00b544c8b5f09adf04b7539a" is_pre_election: true
14:07:22.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.642110 12755 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 122166f9788348b3b2e7f7f0b21d6ae7 in term 0.
14:07:22.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.645148 12817 leader_election.cc:304] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 122166f9788348b3b2e7f7f0b21d6ae7, 8476c8ea00b544c8b5f09adf04b7539a; no voters:
14:07:22.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.646157 12878 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:07:22.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.646600 12878 raft_consensus.cc:491] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:07:22.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.647058 12878 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 0 FOLLOWER]: Advancing to term 1
14:07:22.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.637540 12893 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:22.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.661595 12893 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: This master's current role is: FOLLOWER
14:07:22.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.665890 12878 raft_consensus.cc:513] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:22.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:22.689467 12821 tablet.cc:2378] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:07:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.691004 12755 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8476c8ea00b544c8b5f09adf04b7539a"
14:07:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.691661 12755 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 0 FOLLOWER]: Advancing to term 1
14:07:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.698647 12755 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 122166f9788348b3b2e7f7f0b21d6ae7 in term 1.
14:07:22.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.700513 12817 leader_election.cc:304] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 122166f9788348b3b2e7f7f0b21d6ae7, 8476c8ea00b544c8b5f09adf04b7539a; no voters:
14:07:22.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.702785 12878 leader_election.cc:290] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 1 election: Requested vote from peers 33e3a480d48e42138371de8fb92f5cec (127.12.76.126:37999), 8476c8ea00b544c8b5f09adf04b7539a (127.12.76.125:37303)
14:07:22.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.702832 12900 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Leader election won for term 1
14:07:22.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.704386 12900 raft_consensus.cc:695] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 LEADER]: Becoming Leader. State: Replica: 122166f9788348b3b2e7f7f0b21d6ae7, State: Running, Role: LEADER
14:07:22.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.705277 12900 consensus_queue.cc:237] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:22.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.718042 12878 sys_catalog.cc:455] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 122166f9788348b3b2e7f7f0b21d6ae7. Latest consensus state: current_term: 1 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:22.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.718843 12878 sys_catalog.cc:458] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: This master's current role is: LEADER
14:07:22.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.738981 12905 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:07:22.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.766947 12905 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:07:22.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.770203 12907 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:07:22.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.805039 12755 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 1 FOLLOWER]: Refusing update from remote peer 122166f9788348b3b2e7f7f0b21d6ae7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:07:22.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.806818 12900 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:22.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.775969 12705 master_runner.cc:186] Error getting master registration for 127.12.76.126:37999: OK, instance_id { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" instance_seqno: 1756994838217224 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:07:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.827363 12705 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:22.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:22.829602 12734 tablet.cc:2378] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:07:22.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.850786 12893 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 122166f9788348b3b2e7f7f0b21d6ae7. Latest consensus state: current_term: 1 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:22.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.851496 12893 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: This master's current role is: FOLLOWER
14:07:22.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.875248 12909 mvcc.cc:204] Tried to move back new op lower bound from 7196650876107227136 to 7196650875738333184. Current Snapshot: MvccSnapshot[applied={T|T < 7196650876107227136}]
14:07:22.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.884497 12878 sys_catalog.cc:455] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:22.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.885150 12878 sys_catalog.cc:458] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: This master's current role is: LEADER
14:07:22.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.889009 12905 catalog_manager.cc:1349] Generated new cluster ID: e816491aa42843bfbea6bfc02fb0bf0f
14:07:22.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.889276 12905 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:07:22.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.893189 12893 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:22.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.893741 12893 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: This master's current role is: FOLLOWER
14:07:22.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.929224 12696 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } attempt: 7
14:07:22.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:22 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994838, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.126@KRBTEST.COM for kudu/127.12.76.124@KRBTEST.COM
14:07:22.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.960934 12905 catalog_manager.cc:1372] Generated new certificate authority record
14:07:22.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.962796 12905 catalog_manager.cc:1506] Loading token signing keys...
14:07:22.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.978747 12696 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: Bootstrap starting.
14:07:22.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.995617 12905 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: Generated new TSK 0
14:07:23.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:22.999579 12905 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:07:23.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.000797 12696 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: Neither blocks nor log segments found. Creating new log.
14:07:23.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.003238 12696 log.cc:826] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: Log is configured to *not* fsync() on all Append() calls
14:07:23.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.014514 12696 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: No bootstrap required, opened a new log
14:07:23.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.049898 12696 raft_consensus.cc:357] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:23.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.057344 12696 raft_consensus.cc:383] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:07:23.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.057667 12696 raft_consensus.cc:738] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 33e3a480d48e42138371de8fb92f5cec, State: Initialized, Role: FOLLOWER
14:07:23.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.058545 12696 consensus_queue.cc:260] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:23.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.76.65@KRBTEST.COM; defaulting to no policy
14:07:23.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:23.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.76.65@KRBTEST.COM" created.
14:07:23.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.085564 12696 sys_catalog.cc:564] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: configured and running, proceeding with master startup.
14:07:23.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.086118 12669 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [term 0 FOLLOWER]: Advancing to term 1
14:07:23.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.088970 12671 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "33e3a480d48e42138371de8fb92f5cec" is_pre_election: true
14:07:23.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.090430 12670 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "33e3a480d48e42138371de8fb92f5cec"
14:07:23.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.098325 12669 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [term 1 FOLLOWER]: Refusing update from remote peer 122166f9788348b3b2e7f7f0b21d6ae7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:07:23.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.100056 12878 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:07:23.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.083858 12916 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:23.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.111907 12916 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: This master's current role is: FOLLOWER
14:07:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:23.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:23.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.76.65@KRBTEST.COM; defaulting to no policy
14:07:23.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:23.173853 12650 tablet.cc:2378] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:07:23.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:23.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.76.65@KRBTEST.COM" created.
14:07:23.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.211467 12916 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: SysCatalogTable state changed. Reason: New leader 122166f9788348b3b2e7f7f0b21d6ae7. Latest consensus state: current_term: 1 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.213616 12919 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.214515 12919 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: This master's current role is: FOLLOWER
14:07:23.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.215338 12916 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: This master's current role is: FOLLOWER
14:07:23.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:23.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.65 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:23.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.65 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:23.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:23.229638 12593 mini_kdc.cc:279] Time spent creating service keytab for HTTP/127.12.76.65: real 0.110s user 0.003s sys 0.009s
14:07:23.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.232007 12593 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:23.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:23.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/wal
14:07:23.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/data
14:07:23.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/logs
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/data/info.pb
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.65:0
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.76.65
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.65
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:23.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.65
14:07:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:07:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:07:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:07:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:07:23.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.269101 12937 catalog_manager.cc:1261] Loaded cluster ID: e816491aa42843bfbea6bfc02fb0bf0f
14:07:23.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.269551 12937 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: loading cluster ID for follower catalog manager: success
14:07:23.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.276536 12937 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: acquiring CA information for follower catalog manager: success
14:07:23.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.281827 12937 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:07:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.629606 12878 sys_catalog.cc:455] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:23.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.630347 12878 sys_catalog.cc:458] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: This master's current role is: LEADER
14:07:23.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.793340 12907 catalog_manager.cc:1261] Loaded cluster ID: e816491aa42843bfbea6bfc02fb0bf0f
14:07:23.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.796129 12907 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: loading cluster ID for follower catalog manager: success
14:07:23.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.805886 12907 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: acquiring CA information for follower catalog manager: success
14:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:23.828656 12936 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:23.829 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:23.829253 12936 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:23.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:23.830287 12936 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:23.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.838713 12907 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:07:23.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:23.892071 12936 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:23.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:23.893594 12936 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.76.65
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.958671 12936 tablet_server_runner.cc:78] Tablet server non-default flags:
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/data
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/wal
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.65
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.65:0
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/data/info.pb
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.65
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:23.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.76.65
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/logs
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:23.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.960362 12936 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:23.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:23.961879 12936 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:23.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:23.984279 12947 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:25.378383 12946 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 12936
14:07:26.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:26.156456 12936 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.167s user 0.431s sys 0.913s
14:07:26.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:26.160873 12936 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.172s user 0.432s sys 0.913s
14:07:26.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:23.993764 12948 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:26.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.166261 12936 server_base.cc:1047] running on GCE node
14:07:26.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:26.177546 12956 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:26.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.178882 12936 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:26.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.185927 12936 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:26.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.187359 12936 hybrid_clock.cc:648] HybridClock initialized: now 1756994846187304 us; error 57 us; skew 500 ppm
14:07:26.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.188396 12936 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:26.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:26 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994846, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:26.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.201887 12936 init.cc:377] Logged in from keytab as kudu/127.12.76.65@KRBTEST.COM (short username kudu)
14:07:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.229365 12936 webserver.cc:480] Webserver started at http://127.12.76.65:43503/ using document root <none> and password file <none>
14:07:26.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.230731 12936 fs_manager.cc:362] Metadata directory not provided
14:07:26.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.231129 12936 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:26.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.231796 12936 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.253639 12936 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/data/instance:
14:07:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04630d0051044c1899a4d58a98ccc3ca"
14:07:26.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:26 on dist-test-slave-5mpg"
14:07:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.255266 12936 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/wal/instance:
14:07:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04630d0051044c1899a4d58a98ccc3ca"
14:07:26.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:26 on dist-test-slave-5mpg"
14:07:26.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.276345 12936 fs_manager.cc:696] Time spent creating directory manager: real 0.016s user 0.004s sys 0.003s
14:07:26.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.295244 12962 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:26.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.296666 12936 fs_manager.cc:730] Time spent opening block manager: real 0.016s user 0.005s sys 0.000s
14:07:26.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.297160 12936 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/wal
14:07:26.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "04630d0051044c1899a4d58a98ccc3ca"
14:07:26.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:26 on dist-test-slave-5mpg"
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.297773 12936 fs_report.cc:389] FS layout report
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/wal
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/wal
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/data/data
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:26.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.393064 12936 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:26.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.420010 12936 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:26.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.420855 12936 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:26.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.438165 12936 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:07:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.458045 12936 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:07:26.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.458604 12936 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.001s user 0.001s sys 0.001s
14:07:26.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.459164 12936 ts_tablet_manager.cc:610] Registered 0 tablets
14:07:26.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:26.459594 12936 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:07:27.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.045367 12936 rpc_server.cc:307] RPC server started. Bound to: 127.12.76.65:36065
14:07:27.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.045481 13075 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.76.65:36065 every 8 connection(s)
14:07:27.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.049098 12936 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/data/info.pb
14:07:27.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.056267 12593 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 12936
14:07:27.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.057015 12593 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-0/wal/instance
14:07:27.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.76.66@KRBTEST.COM; defaulting to no policy
14:07:27.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:27.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.76.66@KRBTEST.COM" created.
14:07:27.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:27 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994846, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.65@KRBTEST.COM for kudu/127.12.76.125@KRBTEST.COM
14:07:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.183494 13076 heartbeater.cc:344] Connected to a master server at 127.12.76.125:37303
14:07:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.184026 13076 heartbeater.cc:461] Registering TS with master...
14:07:27.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.185284 13076 heartbeater.cc:507] Master 127.12.76.125:37303 requested a full tablet report, sending...
14:07:27.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:27 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994846, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.65@KRBTEST.COM for kudu/127.12.76.124@KRBTEST.COM
14:07:27.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.189653 12745 ts_manager.cc:194] Registered new tserver with Master: 04630d0051044c1899a4d58a98ccc3ca (127.12.76.65:36065)
14:07:27.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:27.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:27.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:27.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:27.206743 12593 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.12.76.66: real 0.131s user 0.008s sys 0.005s
14:07:27.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.219029 13077 heartbeater.cc:344] Connected to a master server at 127.12.76.124:44285
14:07:27.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.219360 13077 heartbeater.cc:461] Registering TS with master...
14:07:27.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.220266 13077 heartbeater.cc:507] Master 127.12.76.124:44285 requested a full tablet report, sending...
14:07:27.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.223137 12832 ts_manager.cc:194] Registered new tserver with Master: 04630d0051044c1899a4d58a98ccc3ca (127.12.76.65:36065)
14:07:27.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.225886 12832 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.76.65@KRBTEST.COM'} at 127.12.76.65:51671
14:07:27.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.76.66@KRBTEST.COM; defaulting to no policy
14:07:27.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:27.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.76.66@KRBTEST.COM" created.
14:07:27.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:27 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994846, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.65@KRBTEST.COM for kudu/127.12.76.126@KRBTEST.COM
14:07:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.269320 13081 heartbeater.cc:344] Connected to a master server at 127.12.76.126:37999
14:07:27.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.269615 13081 heartbeater.cc:461] Registering TS with master...
14:07:27.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.270205 13081 heartbeater.cc:507] Master 127.12.76.126:37999 requested a full tablet report, sending...
14:07:27.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.273109 12661 ts_manager.cc:194] Registered new tserver with Master: 04630d0051044c1899a4d58a98ccc3ca (127.12.76.65:36065)
14:07:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.66 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:27.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.66 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.293246 12593 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/wal
14:07:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/data
14:07:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:27.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/logs
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/data/info.pb
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.66:0
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.76.66
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.66
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.66
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:07:27.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:27.634305 13090 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:27.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:27.634801 13090 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:27.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:27.635505 13090 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:27.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:27.668098 13090 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:27.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:27.669169 13090 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.76.66
14:07:27.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.706527 13090 tablet_server_runner.cc:78] Tablet server non-default flags:
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/data
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/wal
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.66
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.66:0
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/data/info.pb
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.66
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.76.66
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/logs
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:27.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:07:27.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:27.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:27.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:27.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:27.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:27.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:27.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.708173 13090 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:27.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:27.709421 13090 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:27.721908 13097 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:28.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:28 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994848, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:28.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:28.070606 12640 init.cc:295] Successfully reacquired a new kerberos TGT
14:07:28.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:28.225811 12692 debug-util.cc:398] Leaking SignalData structure 0x7b08000aaa20 after lost signal to thread 12626
14:07:28.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:28.226683 12692 debug-util.cc:398] Leaking SignalData structure 0x7b080009a0c0 after lost signal to thread 12695
14:07:28.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:28.229696 13077 heartbeater.cc:499] Master 127.12.76.124:44285 was elected leader, sending a full tablet report...
14:07:29.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:29.146289 13100 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:29.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:27.722378 13098 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:29.151825 13099 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1430 milliseconds
14:07:29.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.151947 13090 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:29.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.153522 13090 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:29.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.163705 13090 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:29.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.165136 13090 hybrid_clock.cc:648] HybridClock initialized: now 1756994849165095 us; error 42 us; skew 500 ppm
14:07:29.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.166164 13090 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:29.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:29 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994849, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:29.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.173957 13090 init.cc:377] Logged in from keytab as kudu/127.12.76.66@KRBTEST.COM (short username kudu)
14:07:29.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.180819 13090 webserver.cc:480] Webserver started at http://127.12.76.66:34799/ using document root <none> and password file <none>
14:07:29.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.181999 13090 fs_manager.cc:362] Metadata directory not provided
14:07:29.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.182322 13090 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:29.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.182963 13090 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.189560 13090 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/data/instance:
14:07:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a7394d6ec47347ab81c315043013b3d6"
14:07:29.190 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:29 on dist-test-slave-5mpg"
14:07:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.191049 13090 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/wal/instance:
14:07:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a7394d6ec47347ab81c315043013b3d6"
14:07:29.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:29 on dist-test-slave-5mpg"
14:07:29.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.200865 13090 fs_manager.cc:696] Time spent creating directory manager: real 0.009s user 0.010s sys 0.001s
14:07:29.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.206391 13109 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.207401 13090 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.004s
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.207732 13090 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/wal
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a7394d6ec47347ab81c315043013b3d6"
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:29 on dist-test-slave-5mpg"
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.208006 13090 fs_report.cc:389] FS layout report
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/wal
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/wal
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/data/data
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:29.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:29.259 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.258898 13090 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.264225 13090 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:29.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.264710 13090 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.267299 13090 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:07:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.271965 13090 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:07:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.272243 13090 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:07:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.272502 13090 ts_tablet_manager.cc:610] Registered 0 tablets
14:07:29.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.272742 13090 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
14:07:29.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.469250 13090 rpc_server.cc:307] RPC server started. Bound to: 127.12.76.66:45439
14:07:29.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.469305 13222 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.76.66:45439 every 8 connection(s)
14:07:29.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.473174 13090 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/data/info.pb
14:07:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.478048 12593 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 13090
14:07:29.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.478542 12593 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-1/wal/instance
14:07:29.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:29 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994849, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.66@KRBTEST.COM for kudu/127.12.76.125@KRBTEST.COM
14:07:29.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:29 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994849, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.66@KRBTEST.COM for kudu/127.12.76.124@KRBTEST.COM
14:07:29.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.566251 13223 heartbeater.cc:344] Connected to a master server at 127.12.76.125:37303
14:07:29.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.566787 13223 heartbeater.cc:461] Registering TS with master...
14:07:29.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.568179 13223 heartbeater.cc:507] Master 127.12.76.125:37303 requested a full tablet report, sending...
14:07:29.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:29 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994849, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.66@KRBTEST.COM for kudu/127.12.76.126@KRBTEST.COM
14:07:29.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.571730 12745 ts_manager.cc:194] Registered new tserver with Master: a7394d6ec47347ab81c315043013b3d6 (127.12.76.66:45439)
14:07:29.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.12.76.67@KRBTEST.COM; defaulting to no policy
14:07:29.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:29.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.12.76.67@KRBTEST.COM" created.
14:07:29.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.585078 13224 heartbeater.cc:344] Connected to a master server at 127.12.76.124:44285
14:07:29.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.585445 13224 heartbeater.cc:461] Registering TS with master...
14:07:29.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.586406 13224 heartbeater.cc:507] Master 127.12.76.124:44285 requested a full tablet report, sending...
14:07:29.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.588689 12832 ts_manager.cc:194] Registered new tserver with Master: a7394d6ec47347ab81c315043013b3d6 (127.12.76.66:45439)
14:07:29.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.590426 12832 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.76.66@KRBTEST.COM'} at 127.12.76.66:34115
14:07:29.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.592325 13227 heartbeater.cc:344] Connected to a master server at 127.12.76.126:37999
14:07:29.593 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.592731 13227 heartbeater.cc:461] Registering TS with master...
14:07:29.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.593909 13227 heartbeater.cc:507] Master 127.12.76.126:37999 requested a full tablet report, sending...
14:07:29.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.597100 12661 ts_manager.cc:194] Registered new tserver with Master: a7394d6ec47347ab81c315043013b3d6 (127.12.76.66:45439)
14:07:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.12.76.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:29.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:29.605036 12593 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.12.76.67: real 0.113s user 0.001s sys 0.012s
14:07:29.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.12.76.67@KRBTEST.COM; defaulting to no policy
14:07:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:29.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.12.76.67@KRBTEST.COM" created.
14:07:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:07:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.67 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:29.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.12.76.67 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab.
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.641062 12593 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/wal
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/data
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/logs
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/data/info.pb
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:29.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.67:0
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.76.67
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.67
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.67
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:29.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:07:29.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:29.971346 13237 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:29.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:29.971776 13237 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:29.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:29.972329 13237 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:29.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:29 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994849, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:29.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:29.984007 12724 init.cc:295] Successfully reacquired a new kerberos TGT
14:07:30.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:30.005959 13237 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:30.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:30.007189 13237 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.12.76.67
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:30.044792 13237 tablet_server_runner.cc:78] Tablet server non-default flags:
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/data
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/wal
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.67
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.67:0
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:07:30.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/data/info.pb
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.67
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.12.76.67
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/logs
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:30.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:07:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:30.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:30.046448 13237 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:30.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:30.047772 13237 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:30.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:30.067384 13245 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:30.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:30.595655 13224 heartbeater.cc:499] Master 127.12.76.124:44285 was elected leader, sending a full tablet report...
14:07:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:31.465957 13244 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 13237
14:07:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:31.535899 13237 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.472s user 0.497s sys 0.888s
14:07:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:31.536173 13237 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.473s user 0.497s sys 0.888s
14:07:31.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:30.067911 13246 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:31.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:31.554265 13251 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:31.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.555037 13237 server_base.cc:1047] running on GCE node
14:07:31.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.556092 13237 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:31.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.558036 13237 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:31.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.559360 13237 hybrid_clock.cc:648] HybridClock initialized: now 1756994851559310 us; error 40 us; skew 500 ppm
14:07:31.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.560355 13237 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:31.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:31 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994851, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:31.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.566704 13237 init.cc:377] Logged in from keytab as kudu/127.12.76.67@KRBTEST.COM (short username kudu)
14:07:31.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.570892 13237 webserver.cc:480] Webserver started at http://127.12.76.67:44313/ using document root <none> and password file <none>
14:07:31.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.571713 13237 fs_manager.cc:362] Metadata directory not provided
14:07:31.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.571954 13237 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:31.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.572396 13237 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:07:31.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.576442 13237 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/data/instance:
14:07:31.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f511ed40b83f4b0da9b6b1ab5401ef7f"
14:07:31.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:31 on dist-test-slave-5mpg"
14:07:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.577697 13237 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/wal/instance:
14:07:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f511ed40b83f4b0da9b6b1ab5401ef7f"
14:07:31.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:31 on dist-test-slave-5mpg"
14:07:31.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.584903 13237 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.002s
14:07:31.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.589956 13258 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.591046 13237 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.003s
14:07:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.591342 13237 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/wal
14:07:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "f511ed40b83f4b0da9b6b1ab5401ef7f"
14:07:31.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:31 on dist-test-slave-5mpg"
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.591683 13237 fs_report.cc:389] FS layout report
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/wal
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/wal
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/data/data
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:31.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:31.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.640019 13237 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.651202 13237 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:31.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.651623 13237 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.654024 13237 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:07:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.657814 13237 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:07:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.658026 13237 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:07:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.658237 13237 ts_tablet_manager.cc:610] Registered 0 tablets
14:07:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.658365 13237 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:07:31.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.793881 13237 rpc_server.cc:307] RPC server started. Bound to: 127.12.76.67:45505
14:07:31.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.793985 13370 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.76.67:45505 every 8 connection(s)
14:07:31.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.796376 13237 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/data/info.pb
14:07:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.801076 12593 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 13237
14:07:31.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.801627 12593 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/ts-2/wal/instance
14:07:31.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:31 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994851, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.67@KRBTEST.COM for kudu/127.12.76.126@KRBTEST.COM
14:07:31.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:31 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994851, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.67@KRBTEST.COM for kudu/127.12.76.125@KRBTEST.COM
14:07:31.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:31 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994851, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.67@KRBTEST.COM for kudu/127.12.76.124@KRBTEST.COM
14:07:31.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.865432 13371 heartbeater.cc:344] Connected to a master server at 127.12.76.125:37303
14:07:31.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.865919 13371 heartbeater.cc:461] Registering TS with master...
14:07:31.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.867137 13371 heartbeater.cc:507] Master 127.12.76.125:37303 requested a full tablet report, sending...
14:07:31.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.869637 12745 ts_manager.cc:194] Registered new tserver with Master: f511ed40b83f4b0da9b6b1ab5401ef7f (127.12.76.67:45505)
14:07:31.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.874703 13373 heartbeater.cc:344] Connected to a master server at 127.12.76.126:37999
14:07:31.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.875087 13373 heartbeater.cc:461] Registering TS with master...
14:07:31.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.876119 13373 heartbeater.cc:507] Master 127.12.76.126:37999 requested a full tablet report, sending...
14:07:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.877475 13372 heartbeater.cc:344] Connected to a master server at 127.12.76.124:44285
14:07:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.877709 13372 heartbeater.cc:461] Registering TS with master...
14:07:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.878297 13372 heartbeater.cc:507] Master 127.12.76.124:44285 requested a full tablet report, sending...
14:07:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.878501 12661 ts_manager.cc:194] Registered new tserver with Master: f511ed40b83f4b0da9b6b1ab5401ef7f (127.12.76.67:45505)
14:07:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.880730 12832 ts_manager.cc:194] Registered new tserver with Master: f511ed40b83f4b0da9b6b1ab5401ef7f (127.12.76.67:45505)
14:07:31.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.882440 12832 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.12.76.67@KRBTEST.COM'} at 127.12.76.67:42925
14:07:31.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.896494 12593 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:07:31.908 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:31.910 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:07:31.919 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:07:31.919 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:07:31.922 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.12.76.126"
port: 37999
}
bound_http_address {
host: "127.12.76.126"
port: 43093
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.12.76.125"
port: 37303
}
bound_http_address {
host: "127.12.76.125"
port: 42335
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.12.76.124"
port: 44285
}
bound_http_address {
host: "127.12.76.124"
port: 37105
}
}
}
14:07:31.922 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:07:31.924 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.12.76.65"
port: 36065
}
bound_http_address {
host: "127.12.76.65"
port: 43503
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.12.76.66"
port: 45439
}
bound_http_address {
host: "127.12.76.66"
port: 34799
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.12.76.67"
port: 45505
}
bound_http_address {
host: "127.12.76.67"
port: 44313
}
}
}
14:07:31.924 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:07:31.925 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:07:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:31 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994851, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:31.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:07:31.952 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:31.954 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc
14:07:31.960 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:07:31.961 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc
14:07:31.962 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:07:31.965 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
14:07:31.966 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:07:31.966 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:07:31.988 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:31 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994851, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:31.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:07:31.994 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:31.994 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc
14:07:31.995 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:07:31.996 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.76.124:44285
14:07:31.997 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:07:31.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:31.998005 12593 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 12790
14:07:32.022 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:32.022 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.76.125:37303
14:07:32.022 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:07:32.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:32.022586 12593 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 12705
14:07:32.047 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:32.047 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.12.76.126:37999
14:07:32.047 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:07:32.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:32.048187 12593 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 12624
14:07:32.077 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:32.077 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.76.124:44285
14:07:32.081 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:07:32.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:32.083217 12593 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:32.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:32.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal
14:07:32.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/logs
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data/info.pb
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:32.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.124:44285
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.124
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37105
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.124
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:07:32.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:07:32.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.216630 13076 heartbeater.cc:646] Failed to heartbeat to 127.12.76.125:37303 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.12.76.125:37303: connect: Connection refused (error 111)
14:07:32.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.380409 13386 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.380931 13386 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:32.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.381531 13386 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:32.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.409917 13386 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:07:32.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.410125 13386 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:32.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.410535 13386 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:32.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.410858 13386 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:32.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.411175 13386 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:32.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.411479 13386 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:32.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.411744 13386 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38081/jwks.json
14:07:32.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:32.447518 13386 master_runner.cc:386] Master server non-default flags:
14:07:32.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:32.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:32.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:32.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.124
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.124:44285
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data/info.pb
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.124
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37105
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:32.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/logs
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:32.448674 13386 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:32.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:32.449558 13386 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:32.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.460920 13393 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:32.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.598708 13223 heartbeater.cc:646] Failed to heartbeat to 127.12.76.125:37303 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.12.76.125:37303: connect: Connection refused (error 111)
14:07:32.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:32.885598 13372 heartbeater.cc:499] Master 127.12.76.124:44285 was elected leader, sending a full tablet report...
14:07:32.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.913004 13373 heartbeater.cc:646] Failed to heartbeat to 127.12.76.126:37999 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.12.76.126:37999: connect: Connection refused (error 111)
14:07:34.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:33.863694 13392 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 13386
14:07:34.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.016563 13386 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.556s user 0.582s sys 0.941s
14:07:34.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.017143 13386 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.556s user 0.583s sys 0.941s
14:07:34.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:32.461750 13394 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:34.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.019084 13396 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.021267 13395 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1555 milliseconds
14:07:34.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.021274 13386 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:34.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.022555 13386 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:34.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.025048 13386 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:34.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.026340 13386 hybrid_clock.cc:648] HybridClock initialized: now 1756994854026317 us; error 33 us; skew 500 ppm
14:07:34.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.027040 13386 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:34.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:34 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994854, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.124@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:34.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.037065 13386 init.cc:377] Logged in from keytab as kudu/127.12.76.124@KRBTEST.COM (short username kudu)
14:07:34.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.041541 13386 webserver.cc:480] Webserver started at http://127.12.76.124:37105/ using document root <none> and password file <none>
14:07:34.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.042629 13386 fs_manager.cc:362] Metadata directory not provided
14:07:34.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.042920 13386 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:34.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.053155 13386 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
14:07:34.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.058518 13416 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:34.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.059475 13386 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:07:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.059789 13386 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal
14:07:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "122166f9788348b3b2e7f7f0b21d6ae7"
14:07:34.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:22 on dist-test-slave-5mpg"
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.061659 13386 fs_report.cc:389] FS layout report
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/wal
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data/data
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:34.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:34.110 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.110319 13386 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:34.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.118172 13386 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:34.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.118779 13386 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:34.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.198827 13386 rpc_server.cc:307] RPC server started. Bound to: 127.12.76.124:44285
14:07:34.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.198915 13467 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.76.124:44285 every 8 connection(s)
14:07:34.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.201647 13386 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-2/data/info.pb
14:07:34.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.203186 12593 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 13386
14:07:34.203 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:34.204 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.76.125:37303
14:07:34.204 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.205844 12593 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/logs
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data/info.pb
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:34.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.125:37303
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.125
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42335
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:34.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.125
14:07:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:07:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:07:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:07:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:34.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:07:34.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:07:34.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.214896 13468 sys_catalog.cc:263] Verifying existing consensus state
14:07:34.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.221890 13468 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: Bootstrap starting.
14:07:34.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.264297 13468 log.cc:826] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: Log is configured to *not* fsync() on all Append() calls
14:07:34.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.282897 13077 heartbeater.cc:344] Connected to a master server at 127.12.76.124:44285
14:07:34.284 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.284510 13468 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:07:34.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.285451 13468 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: Bootstrap complete.
14:07:34.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.314747 13468 raft_consensus.cc:357] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:34.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.318091 13468 raft_consensus.cc:738] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 122166f9788348b3b2e7f7f0b21d6ae7, State: Initialized, Role: FOLLOWER
14:07:34.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.319048 13468 consensus_queue.cc:260] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:34.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.322341 13475 sys_catalog.cc:455] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:34.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.323283 13475 sys_catalog.cc:458] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: This master's current role is: FOLLOWER
14:07:34.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.324540 13468 sys_catalog.cc:564] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: configured and running, proceeding with master startup.
14:07:34.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.361465 13486 catalog_manager.cc:1261] Loaded cluster ID: e816491aa42843bfbea6bfc02fb0bf0f
14:07:34.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.361711 13486 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: loading cluster ID for follower catalog manager: success
14:07:34.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.369839 13486 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: acquiring CA information for follower catalog manager: success
14:07:34.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.374758 13486 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:07:34.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.367000 13386 master_runner.cc:186] Error getting master registration for 127.12.76.126:37999: Network error: Client connection negotiation failed: client connection to 127.12.76.126:37999: connect: Connection refused (error 111),
14:07:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.392053 13386 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:34.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.542269 13470 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:34.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.542878 13470 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:34.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.543578 13470 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:34.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.586521 13470 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:07:34.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.586813 13470 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:34.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.587276 13470 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:34.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.587615 13470 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:34.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.587929 13470 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:34.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.588248 13470 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:34.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.588488 13470 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38081/jwks.json
14:07:34.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.646951 13470 master_runner.cc:386] Master server non-default flags:
14:07:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data
14:07:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal
14:07:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:07:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:34.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.125
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.125:37303
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data/info.pb
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.125
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=42335
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/logs
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:34.650 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:34.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.648505 13470 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:34.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.649618 13470 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:34.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.656145 13224 heartbeater.cc:344] Connected to a master server at 127.12.76.124:44285
14:07:34.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.665971 13433 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a7394d6ec47347ab81c315043013b3d6" instance_seqno: 1756994849430864) as {username='kudu', principal='kudu/127.12.76.66@KRBTEST.COM'} at 127.12.76.66:40097; Asking this server to re-register.
14:07:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.668179 13224 heartbeater.cc:461] Registering TS with master...
14:07:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:34.671991 13432 ts_manager.cc:194] Registered new tserver with Master: a7394d6ec47347ab81c315043013b3d6 (127.12.76.66:45439)
14:07:34.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.682332 13495 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:35.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:35.040658 13372 heartbeater.cc:344] Connected to a master server at 127.12.76.124:44285
14:07:35.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:35.041345 13372 heartbeater.cc:499] Master 127.12.76.124:44285 was elected leader, sending a full tablet report...
14:07:35.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:35.051385 13432 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "f511ed40b83f4b0da9b6b1ab5401ef7f" instance_seqno: 1756994851763008) as {username='kudu', principal='kudu/127.12.76.67@KRBTEST.COM'} at 127.12.76.67:57067; Asking this server to re-register.
14:07:35.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:35.053905 13372 heartbeater.cc:461] Registering TS with master...
14:07:35.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:35.058135 13432 ts_manager.cc:194] Registered new tserver with Master: f511ed40b83f4b0da9b6b1ab5401ef7f (127.12.76.67:45505)
14:07:35.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:35.290022 13432 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "04630d0051044c1899a4d58a98ccc3ca" instance_seqno: 1756994847002890) as {username='kudu', principal='kudu/127.12.76.65@KRBTEST.COM'} at 127.12.76.65:46281; Asking this server to re-register.
14:07:35.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:35.292774 13077 heartbeater.cc:461] Registering TS with master...
14:07:35.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:35.296394 13432 ts_manager.cc:194] Registered new tserver with Master: 04630d0051044c1899a4d58a98ccc3ca (127.12.76.65:36065)
14:07:36.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.027817 13508 raft_consensus.cc:491] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:07:36.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.028823 13508 raft_consensus.cc:513] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:36.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:36.048345 13419 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.12.76.126:37999: connect: Connection refused (error 111)
14:07:36.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.063850 13508 leader_election.cc:290] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 33e3a480d48e42138371de8fb92f5cec (127.12.76.126:37999), 8476c8ea00b544c8b5f09adf04b7539a (127.12.76.125:37303)
14:07:36.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:36.077155 13418 leader_election.cc:336] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8476c8ea00b544c8b5f09adf04b7539a (127.12.76.125:37303): Network error: Client connection negotiation failed: client connection to 127.12.76.125:37303: connect: Connection refused (error 111)
14:07:36.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:36.082552 13419 leader_election.cc:336] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 33e3a480d48e42138371de8fb92f5cec (127.12.76.126:37999): Network error: Client connection negotiation failed: client connection to 127.12.76.126:37999: connect: Connection refused (error 111)
14:07:36.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.083664 13419 leader_election.cc:304] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 122166f9788348b3b2e7f7f0b21d6ae7; no voters: 33e3a480d48e42138371de8fb92f5cec, 8476c8ea00b544c8b5f09adf04b7539a
14:07:36.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.084980 13508 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:07:36.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:36 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994856, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.65@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:36.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.214107 12958 init.cc:295] Successfully reacquired a new kerberos TGT
14:07:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:36.078819 13494 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 13470
14:07:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:36.423262 13494 kernel_stack_watchdog.cc:198] Thread 13470 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 392ms:
14:07:36.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:07:36.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:07:36.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:07:36.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:07:36.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:34.684238 13496 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:36.430401 13470 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.746s user 0.002s sys 0.000s
14:07:36.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:36.430769 13470 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.746s user 0.002s sys 0.000s
14:07:36.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:36.431685 13498 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:36.433025 13497 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1748 milliseconds
14:07:36.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.433084 13470 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:07:36.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.434762 13470 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:36.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.438369 13470 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:36.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.439862 13470 hybrid_clock.cc:648] HybridClock initialized: now 1756994856439809 us; error 44 us; skew 500 ppm
14:07:36.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.440892 13470 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:36.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:36 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994856, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.125@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:36.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.448020 13470 init.cc:377] Logged in from keytab as kudu/127.12.76.125@KRBTEST.COM (short username kudu)
14:07:36.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.453382 13470 webserver.cc:480] Webserver started at http://127.12.76.125:42335/ using document root <none> and password file <none>
14:07:36.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.454375 13470 fs_manager.cc:362] Metadata directory not provided
14:07:36.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.454631 13470 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:36.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.464668 13470 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.001s
14:07:36.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.470072 13518 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:36.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.471287 13470 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
14:07:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.471657 13470 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal
14:07:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "8476c8ea00b544c8b5f09adf04b7539a"
14:07:36.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:19 on dist-test-slave-5mpg"
14:07:36.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.474280 13470 fs_report.cc:389] FS layout report
14:07:36.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:36.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/wal
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data/data
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:36.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:36.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.538268 13470 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.543895 13470 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:36.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.544391 13470 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:36.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.635423 13470 rpc_server.cc:307] RPC server started. Bound to: 127.12.76.125:37303
14:07:36.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.635499 13569 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.76.125:37303 every 8 connection(s)
14:07:36.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.638849 13470 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-1/data/info.pb
14:07:36.641 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:36.641 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.12.76.126:37999
14:07:36.642 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:07:36.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.639875 12593 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 13470
14:07:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.645303 12593 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu
14:07:36.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal
14:07:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data
14:07:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:07:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:07:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:07:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/logs
14:07:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data/info.pb
14:07:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:36.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:07:36.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.126:37999
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.126
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43093
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.126
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:07:36.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:07:36.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.673779 13570 sys_catalog.cc:263] Verifying existing consensus state
14:07:36.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.685200 13570 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: Bootstrap starting.
14:07:36.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.712806 13223 heartbeater.cc:344] Connected to a master server at 127.12.76.125:37303
14:07:36.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.746767 13570 log.cc:826] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: Log is configured to *not* fsync() on all Append() calls
14:07:36.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.765539 13570 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:07:36.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.766470 13570 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: Bootstrap complete.
14:07:36.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.795447 13570 raft_consensus.cc:357] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:36.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.798583 13570 raft_consensus.cc:738] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8476c8ea00b544c8b5f09adf04b7539a, State: Initialized, Role: FOLLOWER
14:07:36.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.799424 13570 consensus_queue.cc:260] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:36.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.810061 13577 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:36.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.810978 13577 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: This master's current role is: FOLLOWER
14:07:36.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.811918 13570 sys_catalog.cc:564] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: configured and running, proceeding with master startup.
14:07:36.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.865919 13588 catalog_manager.cc:1261] Loaded cluster ID: e816491aa42843bfbea6bfc02fb0bf0f
14:07:36.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.866182 13588 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: loading cluster ID for follower catalog manager: success
14:07:36.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.889559 13588 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: acquiring CA information for follower catalog manager: success
14:07:36.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.894578 13588 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:07:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.859206 13470 master_runner.cc:186] Error getting master registration for 127.12.76.126:37999: Network error: Client connection negotiation failed: client connection to 127.12.76.126:37999: connect: Connection refused (error 111),
14:07:36.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:36.899746 13470 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:37.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.098205 13371 heartbeater.cc:344] Connected to a master server at 127.12.76.125:37303
14:07:37.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.104099 13535 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "f511ed40b83f4b0da9b6b1ab5401ef7f" instance_seqno: 1756994851763008) as {username='kudu', principal='kudu/127.12.76.67@KRBTEST.COM'} at 127.12.76.67:60069; Asking this server to re-register.
14:07:37.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.105928 13371 heartbeater.cc:461] Registering TS with master...
14:07:37.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.108856 13535 ts_manager.cc:194] Registered new tserver with Master: f511ed40b83f4b0da9b6b1ab5401ef7f (127.12.76.67:45505)
14:07:37.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.145825 13573 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:07:37.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.146559 13573 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:07:37.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.147619 13573 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:07:37.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.203575 13573 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:07:37.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.203904 13573 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:07:37.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.204361 13573 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:07:37.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.204725 13573 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:07:37.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.205075 13573 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:07:37.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.205395 13573 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:07:37.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.205636 13573 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:38081/jwks.json
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.271870 13573 master_runner.cc:386] Master server non-default flags:
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.12.76.84:45153
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.12.76.126:37999,127.12.76.125:37303,127.12.76.124:44285
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/krb5kdc/kudu.keytab
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.12.76.126
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:07:37.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.12.76.126:37999
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:38081/jwks.json
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data/info.pb
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.12.76.126
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43093
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/logs
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:07:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:07:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:07:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision db4f8e291cd478c794d4fd4cfb2d6832d781f88f
14:07:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:07:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 04 Sep 2025 13:43:25 UTC on 5fd53c4cbb9d
14:07:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7897
14:07:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:07:37.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.273540 13573 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:07:37.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.274890 13573 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:07:37.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.319921 13076 heartbeater.cc:344] Connected to a master server at 127.12.76.125:37303
14:07:37.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.322105 13599 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:37.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.322913 13535 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "04630d0051044c1899a4d58a98ccc3ca" instance_seqno: 1756994847002890) as {username='kudu', principal='kudu/127.12.76.65@KRBTEST.COM'} at 127.12.76.65:56751; Asking this server to re-register.
14:07:37.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.324393 13076 heartbeater.cc:461] Registering TS with master...
14:07:37.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.328109 13535 ts_manager.cc:194] Registered new tserver with Master: 04630d0051044c1899a4d58a98ccc3ca (127.12.76.65:36065)
14:07:37.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.718709 13535 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a7394d6ec47347ab81c315043013b3d6" instance_seqno: 1756994849430864) as {username='kudu', principal='kudu/127.12.76.66@KRBTEST.COM'} at 127.12.76.66:33565; Asking this server to re-register.
14:07:37.728 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.727279 13223 heartbeater.cc:461] Registering TS with master...
14:07:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.730670 13535 ts_manager.cc:194] Registered new tserver with Master: a7394d6ec47347ab81c315043013b3d6 (127.12.76.66:45439)
14:07:37.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.920452 13606 raft_consensus.cc:491] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:07:37.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.923612 13606 raft_consensus.cc:513] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:37.949 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.948915 13606 leader_election.cc:290] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 33e3a480d48e42138371de8fb92f5cec (127.12.76.126:37999), 8476c8ea00b544c8b5f09adf04b7539a (127.12.76.125:37303)
14:07:37.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.989032 13419 leader_election.cc:336] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 33e3a480d48e42138371de8fb92f5cec (127.12.76.126:37999): Network error: Client connection negotiation failed: client connection to 127.12.76.126:37999: connect: Connection refused (error 111)
14:07:37.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.995508 13545 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8476c8ea00b544c8b5f09adf04b7539a" is_pre_election: true
14:07:37.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:37.996858 13545 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 122166f9788348b3b2e7f7f0b21d6ae7 in term 1.
14:07:38.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.000928 13418 leader_election.cc:304] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 122166f9788348b3b2e7f7f0b21d6ae7, 8476c8ea00b544c8b5f09adf04b7539a; no voters: 33e3a480d48e42138371de8fb92f5cec
14:07:38.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.002470 13606 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:07:38.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.002918 13606 raft_consensus.cc:491] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:07:38.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.003367 13606 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 1 FOLLOWER]: Advancing to term 2
14:07:38.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.031502 13606 raft_consensus.cc:513] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:38.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:38.041293 13419 leader_election.cc:336] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 33e3a480d48e42138371de8fb92f5cec (127.12.76.126:37999): Network error: Client connection negotiation failed: client connection to 127.12.76.126:37999: connect: Connection refused (error 111)
14:07:38.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.042042 13545 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "8476c8ea00b544c8b5f09adf04b7539a"
14:07:38.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.042881 13545 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 1 FOLLOWER]: Advancing to term 2
14:07:38.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.057972 13545 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 122166f9788348b3b2e7f7f0b21d6ae7 in term 2.
14:07:38.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.058800 13606 leader_election.cc:290] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 2 election: Requested vote from peers 33e3a480d48e42138371de8fb92f5cec (127.12.76.126:37999), 8476c8ea00b544c8b5f09adf04b7539a (127.12.76.125:37303)
14:07:38.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.077898 13418 leader_election.cc:304] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 122166f9788348b3b2e7f7f0b21d6ae7, 8476c8ea00b544c8b5f09adf04b7539a; no voters: 33e3a480d48e42138371de8fb92f5cec
14:07:38.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.079705 13606 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 2 FOLLOWER]: Leader election won for term 2
14:07:38.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:38.082719 12436 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac2a0 after lost signal to thread 12309
14:07:38.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:38.083741 12436 debug-util.cc:398] Leaking SignalData structure 0x7b08000ba680 after lost signal to thread 12439
14:07:38.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.111268 13606 raft_consensus.cc:695] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [term 2 LEADER]: Becoming Leader. State: Replica: 122166f9788348b3b2e7f7f0b21d6ae7, State: Running, Role: LEADER
14:07:38.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.112813 13606 consensus_queue.cc:237] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.158041 13613 sys_catalog.cc:455] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 122166f9788348b3b2e7f7f0b21d6ae7. Latest consensus state: current_term: 2 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:38.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.159489 13613 sys_catalog.cc:458] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: This master's current role is: LEADER
14:07:38.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.169353 13616 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:07:38.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.176250 13616 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:07:38.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.195833 13616 catalog_manager.cc:1261] Loaded cluster ID: e816491aa42843bfbea6bfc02fb0bf0f
14:07:38.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.196171 13616 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:07:38.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.200052 13616 catalog_manager.cc:1506] Loading token signing keys...
14:07:38.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.213009 13616 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7: Loaded TSK: 0
14:07:38.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.219610 13616 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:07:38.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.703828 13545 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [term 2 FOLLOWER]: Refusing update from remote peer 122166f9788348b3b2e7f7f0b21d6ae7: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:07:38.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.705536 13619 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
14:07:38.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:38.742674 13419 consensus_peers.cc:489] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 -> Peer 33e3a480d48e42138371de8fb92f5cec (127.12.76.126:37999): Couldn't send request to peer 33e3a480d48e42138371de8fb92f5cec. Status: Network error: Client connection negotiation failed: client connection to 127.12.76.126:37999: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:07:38.754 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.754158 13623 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 122166f9788348b3b2e7f7f0b21d6ae7. Latest consensus state: current_term: 2 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:38.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.754812 13623 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: This master's current role is: FOLLOWER
14:07:38.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.791503 13623 sys_catalog.cc:455] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:38.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.792198 13623 sys_catalog.cc:458] T 00000000000000000000000000000000 P 8476c8ea00b544c8b5f09adf04b7539a [sys.catalog]: This master's current role is: FOLLOWER
14:07:38.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.805091 13627 sys_catalog.cc:455] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:38.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:38.805852 13627 sys_catalog.cc:458] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: This master's current role is: LEADER
14:07:38.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:38.811450 13566 debug-util.cc:398] Leaking SignalData structure 0x7b0800071ec0 after lost signal to thread 13489
14:07:39.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:38.702409 13597 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 13573
14:07:39.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:39.072492 13573 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.752s user 0.551s sys 0.987s
14:07:39.083 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:39.083673 13573 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.763s user 0.551s sys 0.987s
14:07:39.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:37.325735 13601 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:39.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:39.099426 13631 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:07:39.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.109462 13573 server_base.cc:1047] running on GCE node
14:07:39.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.110826 13573 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:07:39.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.121661 13573 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:07:39.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.129352 13573 hybrid_clock.cc:648] HybridClock initialized: now 1756994859129275 us; error 74 us; skew 500 ppm
14:07:39.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.130460 13573 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:07:39.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:39 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994859, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.126@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:39.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.139993 13573 init.cc:377] Logged in from keytab as kudu/127.12.76.126@KRBTEST.COM (short username kudu)
14:07:39.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.168871 13573 webserver.cc:480] Webserver started at http://127.12.76.126:43093/ using document root <none> and password file <none>
14:07:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.170074 13573 fs_manager.cc:362] Metadata directory not provided
14:07:39.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.170368 13573 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:07:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:39 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994859, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.66@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.179368 13105 init.cc:295] Successfully reacquired a new kerberos TGT
14:07:39.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.188877 13573 fs_manager.cc:714] Time spent opening directory manager: real 0.014s user 0.009s sys 0.004s
14:07:39.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.208875 13638 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:07:39.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.211587 13573 fs_manager.cc:730] Time spent opening block manager: real 0.013s user 0.003s sys 0.001s
14:07:39.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.211993 13573 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data,/tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal
14:07:39.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "33e3a480d48e42138371de8fb92f5cec"
14:07:39.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-04 14:07:18 on dist-test-slave-5mpg"
14:07:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.219465 13573 fs_report.cc:389] FS layout report
14:07:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:07:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal
14:07:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/wal
14:07:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data/data
14:07:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:07:39.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:07:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:07:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:07:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:07:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:07:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:07:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:07:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:07:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:07:39.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:07:39.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.315580 13077 heartbeater.cc:499] Master 127.12.76.124:44285 was elected leader, sending a full tablet report...
14:07:39.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.344290 13573 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:07:39.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.358091 13573 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:07:39.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.362187 13573 kserver.cc:163] Server-wide thread pool size limit: 3276
14:07:39.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.394815 13386 master_runner.cc:186] Error getting master registration for 127.12.76.126:37999: Network error: Client connection negotiation failed: client connection to 127.12.76.126:37999: connect: Connection refused (error 111),
14:07:39.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.395232 13386 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:07:39.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.690825 13224 heartbeater.cc:499] Master 127.12.76.124:44285 was elected leader, sending a full tablet report...
14:07:39.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.839082 13573 rpc_server.cc:307] RPC server started. Bound to: 127.12.76.126:37999
14:07:39.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.848457 12593 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu as pid 13573
14:07:39.849 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:07:39.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.847893 13573 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskKowyQv/test-tmp/mini-kudu-cluster9035153836660352820/master-0/data/info.pb
14:07:39.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.857076 13697 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.12.76.126:37999 every 8 connection(s)
14:07:39.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.928840 13699 sys_catalog.cc:263] Verifying existing consensus state
14:07:39.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:39 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994854, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.124@KRBTEST.COM for kudu/127.12.76.126@KRBTEST.COM
14:07:39.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:39.946439 13699 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: Bootstrap starting.
14:07:40.157 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.76.124:44285(127.12.76.124:44285)] connecting to peer
14:07:40.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.160900 13372 heartbeater.cc:499] Master 127.12.76.124:44285 was elected leader, sending a full tablet report...
14:07:40.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.188392 13373 heartbeater.cc:344] Connected to a master server at 127.12.76.126:37999
14:07:40.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.284741 13699 log.cc:826] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: Log is configured to *not* fsync() on all Append() calls
14:07:40.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.331245 13699 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:07:40.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.332218 13699 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: Bootstrap complete.
14:07:40.356 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.76.125:37303(127.12.76.125:37303)] connecting to peer
14:07:40.373 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.12.76.126:37999(127.12.76.126:37999)] connecting to peer
14:07:40.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:40 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994856, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.65@KRBTEST.COM for kudu/127.12.76.126@KRBTEST.COM
14:07:40.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.408098 13081 heartbeater.cc:344] Connected to a master server at 127.12.76.126:37999
14:07:40.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.416488 13699 raft_consensus.cc:357] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:40.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.437274 13699 raft_consensus.cc:738] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 33e3a480d48e42138371de8fb92f5cec, State: Initialized, Role: FOLLOWER
14:07:40.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.441829 13699 consensus_queue.cc:260] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } }
14:07:40.456 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.12.76.125:37303(127.12.76.125:37303)] Successfully connected to peer
14:07:40.483 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.76.124:44285(127.12.76.124:44285)] Successfully connected to peer
14:07:40.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.490805 13709 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:40.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.491667 13709 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: This master's current role is: FOLLOWER
14:07:40.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.496588 13699 sys_catalog.cc:564] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: configured and running, proceeding with master startup.
14:07:40.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.497293 13670 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [term 1 FOLLOWER]: Advancing to term 2
14:07:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.558072 13718 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:40.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.559362 13718 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: This master's current role is: FOLLOWER
14:07:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.562552 13709 sys_catalog.cc:455] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: SysCatalogTable state changed. Reason: New leader 122166f9788348b3b2e7f7f0b21d6ae7. Latest consensus state: current_term: 2 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:40.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.563285 13709 sys_catalog.cc:458] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec [sys.catalog]: This master's current role is: FOLLOWER
14:07:40.654 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.12.76.126:37999(127.12.76.126:37999)] Successfully connected to peer
14:07:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.713495 13725 catalog_manager.cc:1261] Loaded cluster ID: e816491aa42843bfbea6bfc02fb0bf0f
14:07:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.713778 13725 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: loading cluster ID for follower catalog manager: success
14:07:40.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:40 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994859, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.126@KRBTEST.COM for kudu/127.12.76.126@KRBTEST.COM
14:07:40.729 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
14:07:40.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.769994 13725 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: acquiring CA information for follower catalog manager: success
14:07:40.789 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.12.76.125
14:07:40.793 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.12.76.124
14:07:40.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.827102 13725 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 33e3a480d48e42138371de8fb92f5cec: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:07:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.836807 13227 heartbeater.cc:344] Connected to a master server at 127.12.76.126:37999
14:07:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:40 dist-test-slave-5mpg krb5kdc[12605](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994859, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.126@KRBTEST.COM for kudu/127.12.76.125@KRBTEST.COM
14:07:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.853310 13660 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "a7394d6ec47347ab81c315043013b3d6" instance_seqno: 1756994849430864) as {username='kudu', principal='kudu/127.12.76.66@KRBTEST.COM'} at 127.12.76.66:58751; Asking this server to re-register.
14:07:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.855712 13227 heartbeater.cc:461] Registering TS with master...
14:07:40.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.860029 13660 ts_manager.cc:194] Registered new tserver with Master: a7394d6ec47347ab81c315043013b3d6 (127.12.76.66:45439)
14:07:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.947062 13627 sys_catalog.cc:455] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "33e3a480d48e42138371de8fb92f5cec" member_type: VOTER last_known_addr { host: "127.12.76.126" port: 37999 } } peers { permanent_uuid: "8476c8ea00b544c8b5f09adf04b7539a" member_type: VOTER last_known_addr { host: "127.12.76.125" port: 37303 } } peers { permanent_uuid: "122166f9788348b3b2e7f7f0b21d6ae7" member_type: VOTER last_known_addr { host: "127.12.76.124" port: 44285 } } }
14:07:40.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:40.947726 13627 sys_catalog.cc:458] T 00000000000000000000000000000000 P 122166f9788348b3b2e7f7f0b21d6ae7 [sys.catalog]: This master's current role is: LEADER
14:07:41.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.194391 13660 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "f511ed40b83f4b0da9b6b1ab5401ef7f" instance_seqno: 1756994851763008) as {username='kudu', principal='kudu/127.12.76.67@KRBTEST.COM'} at 127.12.76.67:57677; Asking this server to re-register.
14:07:41.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.196075 13373 heartbeater.cc:461] Registering TS with master...
14:07:41.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.199154 13660 ts_manager.cc:194] Registered new tserver with Master: f511ed40b83f4b0da9b6b1ab5401ef7f (127.12.76.67:45505)
14:07:41.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.76.126:37999(127.12.76.126:37999)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:07:41.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.76.126:37999: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:07:41.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:41.213286 13702 negotiation.cc:337] Failed RPC negotiation. Trace:
14:07:41.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.481430 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:58896 (local address 127.12.76.126:37999)
14:07:41.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.482324 (+ 894us) server_negotiation.cc:207] Beginning negotiation
14:07:41.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.482354 (+ 30us) server_negotiation.cc:400] Waiting for connection header
14:07:41.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.657276 (+174922us) server_negotiation.cc:408] Connection header received
14:07:41.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.658120 (+ 844us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:07:41.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.658154 (+ 34us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:07:41.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.658394 (+ 240us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:07:41.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.658566 (+ 172us) server_negotiation.cc:227] Negotiated authn=SASL
14:07:41.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:41.211912 (+553346us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:58896: BlockingRecv error: recv got EOF from 127.0.0.1:58896 (error 108)
14:07:41.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":164}
14:07:41.221 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.76.126:37999(127.12.76.126:37999)] handling channelInactive
14:07:41.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.414553 13660 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "04630d0051044c1899a4d58a98ccc3ca" instance_seqno: 1756994847002890) as {username='kudu', principal='kudu/127.12.76.65@KRBTEST.COM'} at 127.12.76.65:33555; Asking this server to re-register.
14:07:41.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.416049 13081 heartbeater.cc:461] Registering TS with master...
14:07:41.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.418895 13660 ts_manager.cc:194] Registered new tserver with Master: 04630d0051044c1899a4d58a98ccc3ca (127.12.76.65:36065)
14:07:41.437 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.12.76.125:37303(127.12.76.125:37303)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:07:41.441 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.12.76.124:44285(127.12.76.124:44285)] cleaning up while in state NEGOTIATING due to: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:07:41.443 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.12.76.124:44285: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:07:41.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.12.76.125:37303: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:41.451963 13711 negotiation.cc:337] Failed RPC negotiation. Trace:
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.472776 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:51206 (local address 127.12.76.124:44285)
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.481641 (+ 8865us) server_negotiation.cc:207] Beginning negotiation
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.481682 (+ 41us) server_negotiation.cc:400] Waiting for connection header
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.592846 (+111164us) server_negotiation.cc:408] Connection header received
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.654335 (+ 61489us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.654369 (+ 34us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.654599 (+ 230us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.654803 (+ 204us) server_negotiation.cc:227] Negotiated authn=SASL
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:41.205378 (+550575us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:41.207203 (+ 1825us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:41.441217 (+234014us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:41.441852 (+ 635us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:41.450302 (+ 8450us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:51206: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:51206): Cannot send after transport endpoint shutdown (error 108)
14:07:41.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":8239,"thread_start_us":9552,"threads_started":1}
14:07:41.457 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.12.76.124:44285(127.12.76.124:44285)] handling channelInactive
14:07:41.467 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1610), Traces: [0ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.76.124:44285, [492ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.76.125:37303, [508ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.76.126:37999, [1345ms] Sub RPC ConnectToMaster: received response from server master-127.12.76.126:37999: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [1574ms] Sub RPC ConnectToMaster: received response from server master-127.12.76.125:37303: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1578ms] Sub RPC ConnectToMaster: received response from server master-127.12.76.124:44285: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.76.124:44285,127.12.76.125:37303,127.12.76.126:37999). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.76.124:44285,127.12.76.125:37303,127.12.76.126:37999). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14:07:41.477 [DEBUG - kudu-nio-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1620), Traces: [0ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.76.124:44285, [492ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.76.125:37303, [508ms] Sub RPC ConnectToMaster: sending RPC to server master-127.12.76.126:37999, [1345ms] Sub RPC ConnectToMaster: received response from server master-127.12.76.126:37999: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [1574ms] Sub RPC ConnectToMaster: received response from server master-127.12.76.125:37303: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), [1578ms] Sub RPC ConnectToMaster: received response from server master-127.12.76.124:44285: Configuration error: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), deferred=null) with no deferred attached!
14:07:41.483 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.76.126:37999(127.12.76.126:37999)] disconnecting while in state TERMINATED
14:07:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:41.484118 13710 negotiation.cc:337] Failed RPC negotiation. Trace:
14:07:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.448313 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:39402 (local address 127.12.76.125:37303)
14:07:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.486200 (+ 37887us) server_negotiation.cc:207] Beginning negotiation
14:07:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.486227 (+ 27us) server_negotiation.cc:400] Waiting for connection header
14:07:41.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.597024 (+110797us) server_negotiation.cc:408] Connection header received
14:07:41.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.646410 (+ 49386us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
14:07:41.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.646454 (+ 44us) server_negotiation.cc:462] Received NEGOTIATE request from client
14:07:41.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.646769 (+ 315us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
14:07:41.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:40.646971 (+ 202us) server_negotiation.cc:227] Negotiated authn=SASL
14:07:41.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:41.200194 (+553223us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:07:41.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:41.201955 (+ 1761us) server_negotiation.cc:378] Sending TLS_HANDSHAKE NegotiatePB response
14:07:41.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:41.427742 (+225787us) server_negotiation.cc:366] Received TLS_HANDSHAKE NegotiatePB request
14:07:41.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:41.428345 (+ 603us) server_negotiation.cc:663] Negotiated TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 TLSv1.3 Kx=any Au=any Enc=AESGCM(128) Mac=AEAD
14:07:41.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0904 14:07:41.482427 (+ 54082us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:39402: BlockingRecv error: failed to read from TLS socket (remote: 127.0.0.1:39402): Cannot send after transport endpoint shutdown (error 108)
14:07:41.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":37214,"thread_start_us":37916,"threads_started":1}
14:07:41.485 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.76.124:44285(127.12.76.124:44285)] disconnecting while in state TERMINATED
14:07:41.488 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.12.76.125:37303(127.12.76.125:37303)] disconnecting while in state TERMINATED
14:07:41.508 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.12.76.125:37303(127.12.76.125:37303)] handling channelInactive
14:07:41.509 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:07:41.522 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:07:41.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.540263 12593 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 12936
14:07:41.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 04 14:07:41 dist-test-slave-5mpg krb5kdc[12605](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756994861, etypes {rep=17 tkt=17 ses=17}, kudu/127.12.76.67@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:07:41.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.571043 13254 init.cc:295] Successfully reacquired a new kerberos TGT
14:07:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.579998 12593 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 13090
14:07:41.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.623914 12593 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 13237
14:07:41.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.660238 12593 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 13573
14:07:41.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.684666 12593 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 13470
14:07:41.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:41.716002 12593 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 13386
14:07:42.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T14:07:42Z chronyd exiting
14:07:43.938 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:07:43.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:43.939658 11865 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 12144
14:07:43.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:43.965306 11865 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 12305
14:07:43.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:43.993368 11865 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 12450
14:07:44.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:44.018122 11865 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 11877
14:07:44.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:44.046522 11865 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 11947
14:07:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250904 14:07:44.076457 12050 connection.cc:537] client connection to 127.11.150.125:37947 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
14:07:44.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250904 14:07:44.076967 11865 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskKowyQv/build/tsan/bin/kudu with pid 12022
14:07:45.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-04T14:07:45Z chronyd exiting
14:07:46.254 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.76.124:44285,127.12.76.125:37303,127.12.76.126:37999). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.12.76.124:44285,127.12.76.125:37303,127.12.76.126:37999). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT), org.apache.kudu.client.NonRecoverableException: Server requires Kerberos, but this client is not authenticated (missing or expired TGT)]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more