03:02:49.236 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
03:02:49.249 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoRAqye/test-tmp
03:02:49.256 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:02:49.260 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoRAqye/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:02:49.451 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
03:02:49.632 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:02:49.633 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:02:49.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:49.637751 19905 mini_oidc.cc:150] Starting JWKS server
03:02:49.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:49.641460 19905 webserver.cc:492] Webserver started at http://127.0.0.1:35083/ using document root <none> and password file <none>
03:02:49.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:49.642088 19905 mini_oidc.cc:186] Starting OIDC Discovery server
03:02:49.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:49.644147 19905 webserver.cc:492] Webserver started at http://127.0.0.1:43455/ using document root <none> and password file <none>
03:02:49.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:02:49Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:02:49.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:02:49Z Disabled control of system clock
03:02:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:49.711437 19905 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:02:49.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:02:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/wal
03:02:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/data
03:02:49.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:02:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:02:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:02:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:02:49.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:02:49.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:02:49.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/logs
03:02:49.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/data/info.pb
03:02:49.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:02:49.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:02:49.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:02:49.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:02:49.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:02:49.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:02:49.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:02:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:02:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:02:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:02:49.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:02:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:02:49.719 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.126:46751
03:02:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.126
03:02:49.720 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:02:49.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:02:49.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:02:49.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:02:49.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:02:49.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:02:49.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:02:49.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:02:49.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35083/jwks.json with env {}
03:02:50.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:50.016773 19917 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:02:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:50.017508 19917 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:02:50.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:50.018316 19917 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:02:50.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:50.052124 19917 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:02:50.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:50.052533 19917 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:02:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:50.052865 19917 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:02:50.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:50.053198 19917 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:02:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:50.053508 19917 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:02:50.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:50.053756 19917 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35083/jwks.json
03:02:50.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:50.091817 19917 master_runner.cc:386] Master server non-default flags:
03:02:50.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:02:50.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:02:50.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:02:50.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:02:50.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/data
03:02:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/wal
03:02:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:02:50.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:02:50.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:02:50.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:02:50.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:02:50.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.126:46751
03:02:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:02:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:02:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:02:50.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35083/jwks.json
03:02:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:02:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:02:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/data/info.pb
03:02:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.126
03:02:50.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:02:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:02:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:02:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:02:50.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:02:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:02:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/logs
03:02:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:02:50.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:02:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:02:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:02:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:02:50.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:02:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:02:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:02:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:02:50.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:02:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:50.093544 19917 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:02:50.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:50.095885 19917 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:02:50.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:50.114485 19923 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:02:51.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:51.517699 19922 debug-util.cc:398] Leaking SignalData structure 0x7b080002edc0 after lost signal to thread 19917
03:02:51.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:51.832554 19917 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.718s user 0.650s sys 1.066s
03:02:51.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:51.833024 19917 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.719s user 0.650s sys 1.066s
03:02:51.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:50.115331 19924 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:02:51.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:51.836035 19926 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:02:51.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:51.838687 19925 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1718 milliseconds
03:02:51.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.838785 19917 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:02:51.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.840082 19917 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:02:51.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.843180 19917 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:02:51.845 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.845011 19917 hybrid_clock.cc:648] HybridClock initialized: now 1773975771844913 us; error 88 us; skew 500 ppm
03:02:51.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.846140 19917 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:02:51.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.852648 19917 webserver.cc:492] Webserver started at http://127.19.112.126:43903/ using document root <none> and password file <none>
03:02:51.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.853521 19917 fs_manager.cc:362] Metadata directory not provided
03:02:51.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.853706 19917 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:02:51.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.854123 19917 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:02:51.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.858433 19917 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/data/instance:
03:02:51.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "333ccb0515a94abea0b2cc06db04fa9f"
03:02:51.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:02:51 on dist-test-slave-sx75"
03:02:51.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.859982 19917 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/wal/instance:
03:02:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "333ccb0515a94abea0b2cc06db04fa9f"
03:02:51.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:02:51 on dist-test-slave-sx75"
03:02:51.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.867304 19917 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.001s sys 0.006s
03:02:51.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.873054 19933 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:02:51.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.874184 19917 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.005s
03:02:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.874495 19917 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/wal
03:02:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "333ccb0515a94abea0b2cc06db04fa9f"
03:02:51.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:02:51 on dist-test-slave-sx75"
03:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.874934 19917 fs_report.cc:389] FS layout report
03:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/wal
03:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/wal
03:02:51.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/data/data
03:02:51.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:02:51.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:02:51.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:02:51.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:02:51.877 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:02:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:02:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:02:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:02:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:02:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:02:51.878 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:02:51.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.930799 19917 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:02:51.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.932475 19917 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:02:51.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:51.932904 19917 kserver.cc:163] Server-wide thread pool size limit: 3276
03:02:52.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.009531 19917 rpc_server.cc:307] RPC server started. Bound to: 127.19.112.126:46751
03:02:52.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.009611 19984 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.112.126:46751 every 8 connection(s)
03:02:52.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.012445 19917 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/data/info.pb
03:02:52.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.018092 19905 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 19917
03:02:52.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.018651 19905 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-0/wal/instance
03:02:52.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.019649 19985 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:02:52.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.023972 19905 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:02:52.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:02:52.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/wal
03:02:52.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/data
03:02:52.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:02:52.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:02:52.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:02:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:02:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:02:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:02:52.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/logs
03:02:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/data/info.pb
03:02:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:02:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:02:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:02:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:02:52.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:02:52.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:02:52.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:02:52.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:02:52.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:02:52.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:02:52.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:02:52.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:02:52.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.125:35049
03:02:52.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.125
03:02:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:02:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:02:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:02:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:02:52.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:02:52.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:02:52.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:02:52.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:02:52.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35083/jwks.json with env {}
03:02:52.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.037076 19985 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } has no permanent_uuid. Determining permanent_uuid...
03:02:52.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.064719 19985 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } has no permanent_uuid. Determining permanent_uuid...
03:02:52.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.067389 19935 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.112.125:35049: connect: Connection refused (error 111)
03:02:52.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.071887 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.125:35049: Network error: Client connection negotiation failed: client connection to 127.19.112.125:35049: connect: Connection refused (error 111)
03:02:52.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.121358 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } attempt: 1
03:02:52.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.125620 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.125:35049: Network error: Client connection negotiation failed: client connection to 127.19.112.125:35049: connect: Connection refused (error 111)
03:02:52.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.194169 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } attempt: 2
03:02:52.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.198500 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.125:35049: Network error: Client connection negotiation failed: client connection to 127.19.112.125:35049: connect: Connection refused (error 111)
03:02:52.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.290140 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } attempt: 3
03:02:52.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.294462 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.125:35049: Network error: Client connection negotiation failed: client connection to 127.19.112.125:35049: connect: Connection refused (error 111)
03:02:52.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.369056 19987 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:02:52.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.369614 19987 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:02:52.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.370090 19987 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:02:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.401822 19987 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:02:52.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.402099 19987 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:02:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.402302 19987 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:02:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.402487 19987 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:02:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.402801 19987 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:02:52.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.403015 19987 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35083/jwks.json
03:02:52.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.438000 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } attempt: 4
03:02:52.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.442044 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.125:35049: Network error: Client connection negotiation failed: client connection to 127.19.112.125:35049: connect: Connection refused (error 111)
03:02:52.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.442993 19987 master_runner.cc:386] Master server non-default flags:
03:02:52.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:02:52.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:02:52.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:02:52.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:02:52.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/data
03:02:52.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/wal
03:02:52.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:02:52.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:02:52.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:02:52.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:02:52.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:02:52.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.125:35049
03:02:52.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:02:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:02:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:02:52.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35083/jwks.json
03:02:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:02:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:02:52.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/data/info.pb
03:02:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.125
03:02:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:02:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:02:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:02:52.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:02:52.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:02:52.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:02:52.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/logs
03:02:52.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:02:52.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:02:52.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:02:52.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:02:52.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:02:52.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:02:52.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:02:52.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:02:52.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:02:52.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:02:52.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.444288 19987 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:02:52.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.445909 19987 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:02:52.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.458225 19996 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:02:52.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:52.741637 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } attempt: 5
03:02:52.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.749583 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.125:35049: Network error: Client connection negotiation failed: client connection to 127.19.112.125:35049: connect: Connection refused (error 111)
03:02:53.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.297254 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } attempt: 6
03:02:53.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:53.309535 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.125:35049: Network error: Client connection negotiation failed: client connection to 127.19.112.125:35049: connect: Connection refused (error 111)
03:02:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.460700 19997 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:02:53.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:52.463279 19999 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:02:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:53.728672 19998 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1264 milliseconds
03:02:53.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.728802 19987 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:02:53.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.730195 19987 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:02:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.733458 19987 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:02:53.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.734822 19987 hybrid_clock.cc:648] HybridClock initialized: now 1773975773734786 us; error 54 us; skew 500 ppm
03:02:53.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.735622 19987 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:02:53.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.742403 19987 webserver.cc:492] Webserver started at http://127.19.112.125:35761/ using document root <none> and password file <none>
03:02:53.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.743345 19987 fs_manager.cc:362] Metadata directory not provided
03:02:53.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.743559 19987 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:02:53.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.744025 19987 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:02:53.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.748451 19987 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/data/instance:
03:02:53.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bf6292af0d942d99e8705d2e8a9ff4f"
03:02:53.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:02:53 on dist-test-slave-sx75"
03:02:53.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.749560 19987 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/wal/instance:
03:02:53.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bf6292af0d942d99e8705d2e8a9ff4f"
03:02:53.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:02:53 on dist-test-slave-sx75"
03:02:53.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.757335 19987 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.004s
03:02:53.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.763126 20008 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:02:53.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.764401 19987 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
03:02:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.764766 19987 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/wal
03:02:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bf6292af0d942d99e8705d2e8a9ff4f"
03:02:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:02:53 on dist-test-slave-sx75"
03:02:53.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.765182 19987 fs_report.cc:389] FS layout report
03:02:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:02:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/wal
03:02:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/wal
03:02:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/data/data
03:02:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:02:53.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:02:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:02:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:02:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:02:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:02:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:02:53.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:02:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:02:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:02:53.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:02:53.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.825268 19987 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:02:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.827054 19987 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:02:53.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.827610 19987 kserver.cc:163] Server-wide thread pool size limit: 3276
03:02:53.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.905103 19987 rpc_server.cc:307] RPC server started. Bound to: 127.19.112.125:35049
03:02:53.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.905324 20059 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.112.125:35049 every 8 connection(s)
03:02:53.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.908623 19987 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/data/info.pb
03:02:53.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.909617 19905 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 19987
03:02:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.910106 19905 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-1/wal/instance
03:02:53.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.913921 19905 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:02:53.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:02:53.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/wal
03:02:53.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/data
03:02:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:02:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:02:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:02:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:02:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:02:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:02:53.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/logs
03:02:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/data/info.pb
03:02:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:02:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:02:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:02:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:02:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:02:53.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:02:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:02:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:02:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:02:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:02:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:02:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:02:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.124:35183
03:02:53.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.124
03:02:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:02:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:02:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:02:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:02:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:02:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:02:53.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:02:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:02:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35083/jwks.json with env {}
03:02:53.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.917490 20060 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:02:53.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.929140 20060 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } has no permanent_uuid. Determining permanent_uuid...
03:02:53.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.954134 20060 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } has no permanent_uuid. Determining permanent_uuid...
03:02:53.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:53.969157 20060 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } has no permanent_uuid. Determining permanent_uuid...
03:02:53.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:53.971637 20010 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:53.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:53.975955 20060 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:54.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.025571 20060 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 1
03:02:54.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.030387 20060 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:54.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.098906 20060 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 2
03:02:54.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.103219 20060 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:54.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.194723 20060 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 3
03:02:54.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.199065 20060 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:54.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.253794 20062 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:02:54.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.254355 20062 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:02:54.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.254880 20062 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:02:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.287624 20062 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:02:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.287901 20062 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:02:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.288092 20062 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:02:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.288287 20062 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:02:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.288470 20062 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:02:54.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.288607 20062 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:35083/jwks.json
03:02:54.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.325038 20062 master_runner.cc:386] Master server non-default flags:
03:02:54.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:02:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:02:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:02:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:02:54.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/data
03:02:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/wal
03:02:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:02:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:02:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:02:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:02:54.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:02:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.124:35183
03:02:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:02:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:02:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:02:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:35083/jwks.json
03:02:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:02:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:02:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/data/info.pb
03:02:54.328 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.124
03:02:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:02:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:02:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:02:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:02:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:02:54.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:02:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/logs
03:02:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:02:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:02:54.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:02:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:02:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:02:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:02:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:02:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:02:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:02:54.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:02:54.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.326460 20062 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:02:54.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.328790 20062 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:02:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.342484 20072 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:02:54.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.342554 20060 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 4
03:02:54.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.348637 20060 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:54.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.370134 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } attempt: 7
03:02:54.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.395973 19985 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } has no permanent_uuid. Determining permanent_uuid...
03:02:54.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.425189 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:54.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.483814 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 1
03:02:54.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.492383 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:54.575 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.574039 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 2
03:02:54.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.583199 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:54.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.648118 20060 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 5
03:02:54.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.656278 20060 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:54.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.668820 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 3
03:02:54.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.676508 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:54.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:54.817082 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 4
03:02:54.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.824983 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:55.109 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:55.108531 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 5
03:02:55.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:55.118364 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:55.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:55.203832 20060 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 6
03:02:55.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:55.242940 20060 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:55.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:55.671774 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 6
03:02:55.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:55.753017 19985 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.19.112.124:35183: Network error: Client connection negotiation failed: client connection to 127.19.112.124:35183: connect: Connection refused (error 111)
03:02:56.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:55.747417 20071 debug-util.cc:398] Leaking SignalData structure 0x7b080002edc0 after lost signal to thread 20062
03:02:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.117319 20071 kernel_stack_watchdog.cc:198] Thread 20062 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 399ms:
03:02:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:02:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:02:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:02:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:02:56.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:54.343708 20073 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:02:56.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.124217 20062 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.779s user 0.000s sys 0.006s
03:02:56.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.124600 20062 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.780s user 0.000s sys 0.006s
03:02:56.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.124792 20074 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1775 milliseconds
03:02:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.126612 20076 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:02:56.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.126682 20062 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:02:56.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.128010 20062 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:02:56.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.131074 20062 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:02:56.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.132524 20062 hybrid_clock.cc:648] HybridClock initialized: now 1773975776132478 us; error 55 us; skew 500 ppm
03:02:56.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.133359 20062 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:02:56.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.139645 20062 webserver.cc:492] Webserver started at http://127.19.112.124:46081/ using document root <none> and password file <none>
03:02:56.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.140687 20062 fs_manager.cc:362] Metadata directory not provided
03:02:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.140887 20062 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:02:56.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.141278 20062 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:02:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.145954 20062 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/data/instance:
03:02:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bb35da3b19f4e86920e431f9c38bc93"
03:02:56.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:02:56 on dist-test-slave-sx75"
03:02:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.147181 20062 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/wal/instance:
03:02:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bb35da3b19f4e86920e431f9c38bc93"
03:02:56.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:02:56 on dist-test-slave-sx75"
03:02:56.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.154565 20062 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
03:02:56.160 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.160408 20089 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:02:56.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.161602 20062 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
03:02:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.161891 20062 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/wal
03:02:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bb35da3b19f4e86920e431f9c38bc93"
03:02:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:02:56 on dist-test-slave-sx75"
03:02:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.162262 20062 fs_report.cc:389] FS layout report
03:02:56.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/wal
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/wal
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/data/data
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:02:56.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:02:56.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.216899 20062 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:02:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.218686 20062 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:02:56.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.219182 20062 kserver.cc:163] Server-wide thread pool size limit: 3276
03:02:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.299189 20062 rpc_server.cc:307] RPC server started. Bound to: 127.19.112.124:35183
03:02:56.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.299329 20140 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.112.124:35183 every 8 connection(s)
03:02:56.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.302294 20062 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/data/info.pb
03:02:56.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.303542 20060 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 7
03:02:56.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.307417 19905 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 20062
03:02:56.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.307911 19905 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/master-2/wal/instance
03:02:56.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.311470 20141 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:02:56.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.344910 20141 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } has no permanent_uuid. Determining permanent_uuid...
03:02:56.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.394513 20141 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } has no permanent_uuid. Determining permanent_uuid...
03:02:56.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.401213 20060 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f: Bootstrap starting.
03:02:56.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.408279 20060 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f: Neither blocks nor log segments found. Creating new log.
03:02:56.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.408334 20141 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } has no permanent_uuid. Determining permanent_uuid...
03:02:56.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.410670 20060 log.cc:826] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f: Log is configured to *not* fsync() on all Append() calls
03:02:56.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.416252 20060 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f: No bootstrap required, opened a new log
03:02:56.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.429215 20141 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93: Bootstrap starting.
03:02:56.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.435546 20141 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93: Neither blocks nor log segments found. Creating new log.
03:02:56.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.437368 20141 log.cc:826] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93: Log is configured to *not* fsync() on all Append() calls
03:02:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.437160 20060 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } }
03:02:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.438016 20060 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:02:56.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.438437 20060 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2bf6292af0d942d99e8705d2e8a9ff4f, State: Initialized, Role: FOLLOWER
03:02:56.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.439651 20060 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } }
03:02:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.443822 20141 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93: No bootstrap required, opened a new log
03:02:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.443213 20154 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } } }
03:02:56.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.444083 20154 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [sys.catalog]: This master's current role is: FOLLOWER
03:02:56.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.445081 20060 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [sys.catalog]: configured and running, proceeding with master startup.
03:02:56.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.466029 20141 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } }
03:02:56.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.466761 20141 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:02:56.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.467041 20141 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2bb35da3b19f4e86920e431f9c38bc93, State: Initialized, Role: FOLLOWER
03:02:56.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.467909 20141 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } }
03:02:56.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.471374 20164 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } } }
03:02:56.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.472537 20164 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [sys.catalog]: This master's current role is: FOLLOWER
03:02:56.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.473357 20141 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [sys.catalog]: configured and running, proceeding with master startup.
03:02:56.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.475009 20095 tablet.cc:2379] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:02:56.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.496222 20166 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:02:56.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.497152 20166 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:02:56.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.483616 19987 master_runner.cc:186] Error getting master registration for 127.19.112.126:46751: OK, instance_id { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" instance_seqno: 1773975771988144 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:02:56.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.516206 19987 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:02:56.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.528645 20178 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:02:56.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.529562 20178 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:02:56.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.510519 20062 master_runner.cc:186] Error getting master registration for 127.19.112.126:46751: OK, instance_id { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" instance_seqno: 1773975771988144 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:02:56.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.538852 20062 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:02:56.587 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.587278 20014 tablet.cc:2379] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:02:56.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.686395 20164 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:02:56.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.687058 20164 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } }
03:02:56.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.689846 20164 leader_election.cc:290] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 333ccb0515a94abea0b2cc06db04fa9f (127.19.112.126:46751), 2bf6292af0d942d99e8705d2e8a9ff4f (127.19.112.125:35049)
03:02:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.695111 20035 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2bb35da3b19f4e86920e431f9c38bc93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" is_pre_election: true
03:02:56.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.696085 20035 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2bb35da3b19f4e86920e431f9c38bc93 in term 0.
03:02:56.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.698686 20091 leader_election.cc:304] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2bb35da3b19f4e86920e431f9c38bc93, 2bf6292af0d942d99e8705d2e8a9ff4f; no voters:
03:02:56.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.699956 20164 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [term 0 FOLLOWER]: Leader pre-election won for term 1
03:02:56.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.700363 20164 raft_consensus.cc:493] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:02:56.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.701927 20164 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [term 0 FOLLOWER]: Advancing to term 1
03:02:56.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.709232 20164 raft_consensus.cc:515] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } }
03:02:56.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.711270 20164 leader_election.cc:290] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [CANDIDATE]: Term 1 election: Requested vote from peers 333ccb0515a94abea0b2cc06db04fa9f (127.19.112.126:46751), 2bf6292af0d942d99e8705d2e8a9ff4f (127.19.112.125:35049)
03:02:56.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.713716 20035 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2bb35da3b19f4e86920e431f9c38bc93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f"
03:02:56.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.714284 20035 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [term 0 FOLLOWER]: Advancing to term 1
03:02:56.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.721585 20035 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2bb35da3b19f4e86920e431f9c38bc93 in term 1.
03:02:56.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:56.731503 19981 debug-util.cc:398] Leaking SignalData structure 0x7b080003cc00 after lost signal to thread 19918
03:02:56.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.738633 20091 leader_election.cc:304] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2bb35da3b19f4e86920e431f9c38bc93, 2bf6292af0d942d99e8705d2e8a9ff4f; no voters:
03:02:56.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.741058 20164 raft_consensus.cc:2804] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [term 1 FOLLOWER]: Leader election won for term 1
03:02:56.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.743436 20164 raft_consensus.cc:697] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [term 1 LEADER]: Becoming Leader. State: Replica: 2bb35da3b19f4e86920e431f9c38bc93, State: Running, Role: LEADER
03:02:56.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.744714 20164 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } }
03:02:56.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.772048 20180 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2bb35da3b19f4e86920e431f9c38bc93. Latest consensus state: current_term: 1 leader_uuid: "2bb35da3b19f4e86920e431f9c38bc93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } } }
03:02:56.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.774750 20180 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [sys.catalog]: This master's current role is: LEADER
03:02:56.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.787532 19985 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } attempt: 7
03:02:56.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.797366 20182 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:02:56.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.806051 20182 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:02:56.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.838766 19985 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f: Bootstrap starting.
03:02:56.839 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.839069 20035 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [term 1 FOLLOWER]: Refusing update from remote peer 2bb35da3b19f4e86920e431f9c38bc93: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:02:56.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.841681 20180 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:02:56.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.846786 19985 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f: Neither blocks nor log segments found. Creating new log.
03:02:56.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.849249 19985 log.cc:826] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f: Log is configured to *not* fsync() on all Append() calls
03:02:56.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.880515 20154 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2bb35da3b19f4e86920e431f9c38bc93. Latest consensus state: current_term: 1 leader_uuid: "2bb35da3b19f4e86920e431f9c38bc93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } } }
03:02:56.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.881297 20154 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [sys.catalog]: This master's current role is: FOLLOWER
03:02:56.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.884908 19985 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f: No bootstrap required, opened a new log
03:02:56.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.930166 19985 raft_consensus.cc:359] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } }
03:02:56.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.931669 19985 raft_consensus.cc:385] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:02:56.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.932297 19985 raft_consensus.cc:740] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 333ccb0515a94abea0b2cc06db04fa9f, State: Initialized, Role: FOLLOWER
03:02:56.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.933604 19985 consensus_queue.cc:260] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } }
03:02:56.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.941247 20154 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2bb35da3b19f4e86920e431f9c38bc93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } } }
03:02:56.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.942059 20154 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f [sys.catalog]: This master's current role is: FOLLOWER
03:02:56.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.956663 20180 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2bb35da3b19f4e86920e431f9c38bc93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } } }
03:02:56.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.957803 20180 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [sys.catalog]: This master's current role is: LEADER
03:02:56.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.958312 20182 catalog_manager.cc:1357] Generated new cluster ID: 9f31491edf8e47b3881188c5308c6e36
03:02:56.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.958642 20182 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:02:57.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:56.999827 20190 sys_catalog.cc:455] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } } }
03:02:57.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.001343 20190 sys_catalog.cc:458] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [sys.catalog]: This master's current role is: FOLLOWER
03:02:57.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.002012 19985 sys_catalog.cc:565] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [sys.catalog]: configured and running, proceeding with master startup.
03:02:57.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.002193 19960 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2bb35da3b19f4e86920e431f9c38bc93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "333ccb0515a94abea0b2cc06db04fa9f" is_pre_election: true
03:02:57.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.003082 19960 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2bb35da3b19f4e86920e431f9c38bc93 in term 0.
03:02:57.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.004452 19958 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [term 0 FOLLOWER]: Advancing to term 1
03:02:57.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.012332 19959 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2bb35da3b19f4e86920e431f9c38bc93" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "333ccb0515a94abea0b2cc06db04fa9f"
03:02:57.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.017969 19958 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [term 1 FOLLOWER]: Refusing update from remote peer 2bb35da3b19f4e86920e431f9c38bc93: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:02:57.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.032568 20164 consensus_queue.cc:1048] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [LEADER]: Connected to new peer: Peer: permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:02:57.050 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.049619 20182 catalog_manager.cc:1380] Generated new certificate authority record
03:02:57.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.051955 20182 catalog_manager.cc:1514] Loading token signing keys...
03:02:57.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.094280 20190 sys_catalog.cc:455] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2bb35da3b19f4e86920e431f9c38bc93. Latest consensus state: current_term: 1 leader_uuid: "2bb35da3b19f4e86920e431f9c38bc93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } } }
03:02:57.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.095544 20190 sys_catalog.cc:458] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [sys.catalog]: This master's current role is: FOLLOWER
03:02:57.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.097061 20202 sys_catalog.cc:455] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "2bb35da3b19f4e86920e431f9c38bc93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } } }
03:02:57.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.098163 20202 sys_catalog.cc:458] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f [sys.catalog]: This master's current role is: FOLLOWER
03:02:57.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.101317 20180 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "2bb35da3b19f4e86920e431f9c38bc93" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "333ccb0515a94abea0b2cc06db04fa9f" member_type: VOTER last_known_addr { host: "127.19.112.126" port: 46751 } } peers { permanent_uuid: "2bf6292af0d942d99e8705d2e8a9ff4f" member_type: VOTER last_known_addr { host: "127.19.112.125" port: 35049 } } peers { permanent_uuid: "2bb35da3b19f4e86920e431f9c38bc93" member_type: VOTER last_known_addr { host: "127.19.112.124" port: 35183 } } }
03:02:57.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.101986 20180 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93 [sys.catalog]: This master's current role is: LEADER
03:02:57.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.146440 20182 catalog_manager.cc:6044] T 00000000000000000000000000000000 P 2bb35da3b19f4e86920e431f9c38bc93: Generated new TSK 0
03:02:57.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.148130 20182 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:02:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.179610 19905 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:02:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:02:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/wal
03:02:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/data
03:02:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:02:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:02:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:02:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:02:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:02:57.180 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:02:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/logs
03:02:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/data/info.pb
03:02:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:02:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:02:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:02:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:02:57.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.65:0
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.112.65
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.65
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:02:57.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:02:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:02:57.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:02:57.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.191588 20207 catalog_manager.cc:1269] Loaded cluster ID: 9f31491edf8e47b3881188c5308c6e36
03:02:57.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.191903 20207 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f: loading cluster ID for follower catalog manager: success
03:02:57.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.219596 20207 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f: acquiring CA information for follower catalog manager: success
03:02:57.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.231082 20207 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 333ccb0515a94abea0b2cc06db04fa9f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:02:57.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.503746 20166 catalog_manager.cc:1269] Loaded cluster ID: 9f31491edf8e47b3881188c5308c6e36
03:02:57.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.504570 20166 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f: loading cluster ID for follower catalog manager: success
03:02:57.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.513192 20166 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f: acquiring CA information for follower catalog manager: success
03:02:57.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.544724 20166 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 2bf6292af0d942d99e8705d2e8a9ff4f: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:02:57.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:57.769311 20210 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:02:57.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:57.770418 20210 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:02:57.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:57.771523 20210 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:02:57.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:57.832755 20210 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:02:57.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:57.834766 20210 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.112.65
03:02:57.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.904642 20210 tablet_server_runner.cc:78] Tablet server non-default flags:
03:02:57.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:02:57.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:02:57.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:02:57.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:02:57.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/data
03:02:57.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/wal
03:02:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:02:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:02:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.65:0
03:02:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:02:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:02:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:02:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/data/info.pb
03:02:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.65
03:02:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:02:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:02:57.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:02:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:02:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:02:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:02:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:02:57.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.112.65
03:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/logs
03:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:02:57.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:02:57.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.911718 20210 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:02:57.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:02:57.914683 20210 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:02:57.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:57.978268 20216 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:00.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:59.364945 20215 debug-util.cc:398] Leaking SignalData structure 0x7b080002eda0 after lost signal to thread 20210
03:03:00.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:00.517624 20215 kernel_stack_watchdog.cc:198] Thread 20210 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 377ms:
03:03:00.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:03:00.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:03:00.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:03:00.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:03:00.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:02:57.985435 20217 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:00.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:00.519778 20210 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.536s user 0.510s sys 0.891s
03:03:00.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:00.521572 20210 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.538s user 0.510s sys 0.891s
03:03:00.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.529738 20210 server_base.cc:1061] running on GCE node
03:03:00.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:00.530244 20223 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:00.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.531737 20210 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:00.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.547461 20210 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:00.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.548983 20210 hybrid_clock.cc:648] HybridClock initialized: now 1773975780548937 us; error 55 us; skew 500 ppm
03:03:00.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.550163 20210 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:00.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.580122 20210 webserver.cc:492] Webserver started at http://127.19.112.65:44043/ using document root <none> and password file <none>
03:03:00.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.581517 20210 fs_manager.cc:362] Metadata directory not provided
03:03:00.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.581826 20210 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:00.582 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.582509 20210 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:03:00.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.589226 20210 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/data/instance:
03:03:00.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10406ccbf5364b4a8cfbe15bbf0632a9"
03:03:00.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:00 on dist-test-slave-sx75"
03:03:00.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.590833 20210 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/wal/instance:
03:03:00.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10406ccbf5364b4a8cfbe15bbf0632a9"
03:03:00.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:00 on dist-test-slave-sx75"
03:03:00.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.610419 20210 fs_manager.cc:696] Time spent creating directory manager: real 0.019s user 0.004s sys 0.013s
03:03:00.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.624531 20228 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.625914 20210 fs_manager.cc:730] Time spent opening block manager: real 0.010s user 0.001s sys 0.008s
03:03:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.626318 20210 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/wal
03:03:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "10406ccbf5364b4a8cfbe15bbf0632a9"
03:03:00.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:00 on dist-test-slave-sx75"
03:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.626814 20210 fs_report.cc:389] FS layout report
03:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/wal
03:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/wal
03:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/data/data
03:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:00.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:00.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:00.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.709144 20210 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:00.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.711555 20210 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:00.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.712255 20210 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:00.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.721997 20210 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:03:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.734023 20210 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:03:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.734326 20210 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:03:00.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.734668 20210 ts_tablet_manager.cc:616] Registered 0 tablets
03:03:00.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:00.734912 20210 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
03:03:01.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.130474 20210 rpc_server.cc:307] RPC server started. Bound to: 127.19.112.65:35661
03:03:01.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.130631 20343 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.112.65:35661 every 8 connection(s)
03:03:01.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.133761 20210 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/data/info.pb
03:03:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.137413 19905 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 20210
03:03:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.138094 19905 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-0/wal/instance
03:03:01.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.161449 19905 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/wal
03:03:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/data
03:03:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/logs
03:03:01.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/data/info.pb
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.66:0
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.112.66
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.66
03:03:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:01.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:03:01.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:03:01.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:01.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:01.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:03:01.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.202783 20346 heartbeater.cc:344] Connected to a master server at 127.19.112.126:46751
03:03:01.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.203346 20346 heartbeater.cc:461] Registering TS with master...
03:03:01.205 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.204773 20346 heartbeater.cc:507] Master 127.19.112.126:46751 requested a full tablet report, sending...
03:03:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.207201 20345 heartbeater.cc:344] Connected to a master server at 127.19.112.125:35049
03:03:01.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.207577 20345 heartbeater.cc:461] Registering TS with master...
03:03:01.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.208570 20345 heartbeater.cc:507] Master 127.19.112.125:35049 requested a full tablet report, sending...
03:03:01.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.208467 19950 ts_manager.cc:194] Registered new tserver with Master: 10406ccbf5364b4a8cfbe15bbf0632a9 (127.19.112.65:35661)
03:03:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.211292 20344 heartbeater.cc:344] Connected to a master server at 127.19.112.124:35183
03:03:01.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.211604 20344 heartbeater.cc:461] Registering TS with master...
03:03:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.211589 20025 ts_manager.cc:194] Registered new tserver with Master: 10406ccbf5364b4a8cfbe15bbf0632a9 (127.19.112.65:35661)
03:03:01.212 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.212636 20344 heartbeater.cc:507] Master 127.19.112.124:35183 requested a full tablet report, sending...
03:03:01.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.215353 20106 ts_manager.cc:194] Registered new tserver with Master: 10406ccbf5364b4a8cfbe15bbf0632a9 (127.19.112.65:35661)
03:03:01.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.218008 20106 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.19.112.65:54971
03:03:01.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:01.540560 20354 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:01.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:01.541229 20354 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:01.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:01.541998 20354 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:01.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:01.595587 20354 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:01.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:01.596400 20354 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.112.66
03:03:01.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.633556 20354 tablet_server_runner.cc:78] Tablet server non-default flags:
03:03:01.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:01.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:03:01.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/data
03:03:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/wal
03:03:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.66:0
03:03:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/data/info.pb
03:03:01.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.66
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.112.66
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/logs
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:01.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:01.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.634951 20354 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:01.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:01.636505 20354 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:01.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:01.652173 20360 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:02.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:02.222333 20344 heartbeater.cc:499] Master 127.19.112.124:35183 was elected leader, sending a full tablet report...
03:03:03.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:01.652585 20361 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:03.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:03.021353 20354 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.371s user 0.534s sys 0.830s
03:03:03.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:03.021763 20354 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.372s user 0.534s sys 0.830s
03:03:03.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.029995 20354 server_base.cc:1061] running on GCE node
03:03:03.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:03.030395 20366 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:03.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.031752 20354 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:03.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.034029 20354 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:03.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.035501 20354 hybrid_clock.cc:648] HybridClock initialized: now 1773975783035448 us; error 59 us; skew 500 ppm
03:03:03.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.036564 20354 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:03.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.044013 20354 webserver.cc:492] Webserver started at http://127.19.112.66:39151/ using document root <none> and password file <none>
03:03:03.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.045203 20354 fs_manager.cc:362] Metadata directory not provided
03:03:03.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.045483 20354 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:03.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.046017 20354 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:03:03.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.050594 20354 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/data/instance:
03:03:03.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26ea52325aea4d07aae6866510cdea5b"
03:03:03.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:03 on dist-test-slave-sx75"
03:03:03.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.051977 20354 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/wal/instance:
03:03:03.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26ea52325aea4d07aae6866510cdea5b"
03:03:03.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:03 on dist-test-slave-sx75"
03:03:03.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.060122 20354 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.002s sys 0.006s
03:03:03.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.067538 20371 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:03.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.068820 20354 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.001s sys 0.003s
03:03:03.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.069175 20354 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/wal
03:03:03.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "26ea52325aea4d07aae6866510cdea5b"
03:03:03.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:03 on dist-test-slave-sx75"
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.069603 20354 fs_report.cc:389] FS layout report
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/wal
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/wal
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/data/data
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:03.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:03.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:03.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.136989 20354 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.138728 20354 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:03.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.139292 20354 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:03.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.142050 20354 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:03:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.146413 20354 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:03:03.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.146744 20354 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
03:03:03.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.147073 20354 ts_tablet_manager.cc:616] Registered 0 tablets
03:03:03.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.147274 20354 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:03:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.298244 20354 rpc_server.cc:307] RPC server started. Bound to: 127.19.112.66:40557
03:03:03.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.298343 20485 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.112.66:40557 every 8 connection(s)
03:03:03.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.302211 20354 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/data/info.pb
03:03:03.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.312227 19905 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 20354
03:03:03.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.312670 19905 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-1/wal/instance
03:03:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.342268 19905 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:03.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/wal
03:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/data
03:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:03.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/logs
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/data/info.pb
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.67:0
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.112.67
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.67
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:03.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
03:03:03.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.370121 20487 heartbeater.cc:344] Connected to a master server at 127.19.112.125:35049
03:03:03.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.370664 20487 heartbeater.cc:461] Registering TS with master...
03:03:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.371515 20488 heartbeater.cc:344] Connected to a master server at 127.19.112.126:46751
03:03:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.371904 20488 heartbeater.cc:461] Registering TS with master...
03:03:03.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.372084 20487 heartbeater.cc:507] Master 127.19.112.125:35049 requested a full tablet report, sending...
03:03:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.373131 20488 heartbeater.cc:507] Master 127.19.112.126:46751 requested a full tablet report, sending...
03:03:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.373241 20486 heartbeater.cc:344] Connected to a master server at 127.19.112.124:35183
03:03:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.373617 20486 heartbeater.cc:461] Registering TS with master...
03:03:03.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.374506 20486 heartbeater.cc:507] Master 127.19.112.124:35183 requested a full tablet report, sending...
03:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.376500 20025 ts_manager.cc:194] Registered new tserver with Master: 26ea52325aea4d07aae6866510cdea5b (127.19.112.66:40557)
03:03:03.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.376722 20106 ts_manager.cc:194] Registered new tserver with Master: 26ea52325aea4d07aae6866510cdea5b (127.19.112.66:40557)
03:03:03.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.378496 20106 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.19.112.66:59305
03:03:03.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.381840 19950 ts_manager.cc:194] Registered new tserver with Master: 26ea52325aea4d07aae6866510cdea5b (127.19.112.66:40557)
03:03:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:03.670953 20496 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:03.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:03.671474 20496 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:03.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:03.671900 20496 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:03.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:03.703332 20496 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:03.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:03.704247 20496 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.19.112.67
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.739723 20496 tablet_server_runner.cc:78] Tablet server non-default flags:
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.19.112.84:41169
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/data
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/wal
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.19.112.67:0
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/data/info.pb
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.19.112.67
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.19.112.126:46751,127.19.112.125:35049,127.19.112.124:35183
03:03:03.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.19.112.67
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/logs
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:03.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.741195 20496 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:03.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:03.742874 20496 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:03.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:03.756457 20502 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:04.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:04.382983 20486 heartbeater.cc:499] Master 127.19.112.124:35183 was elected leader, sending a full tablet report...
03:03:05.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:03.757481 20503 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:05.027837 20505 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:05.030853 20504 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1268 milliseconds
03:03:05.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.031000 20496 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:03:05.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.032485 20496 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:05.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.034855 20496 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:05.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.036316 20496 hybrid_clock.cc:648] HybridClock initialized: now 1773975785036281 us; error 40 us; skew 500 ppm
03:03:05.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.037237 20496 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:05.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.044260 20496 webserver.cc:492] Webserver started at http://127.19.112.67:46013/ using document root <none> and password file <none>
03:03:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.045281 20496 fs_manager.cc:362] Metadata directory not provided
03:03:05.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.045696 20496 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:05.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.046268 20496 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:03:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.051288 20496 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/data/instance:
03:03:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "792eb096672c4b0a94b91b69e4ec20b6"
03:03:05.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:05 on dist-test-slave-sx75"
03:03:05.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.052554 20496 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/wal/instance:
03:03:05.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "792eb096672c4b0a94b91b69e4ec20b6"
03:03:05.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:05 on dist-test-slave-sx75"
03:03:05.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.060376 20496 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.003s
03:03:05.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.066392 20513 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:05.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.067600 20496 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
03:03:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.067965 20496 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/wal
03:03:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "792eb096672c4b0a94b91b69e4ec20b6"
03:03:05.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:05 on dist-test-slave-sx75"
03:03:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.068397 20496 fs_report.cc:389] FS layout report
03:03:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/wal
03:03:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/wal
03:03:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/data/data
03:03:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:05.069 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:05.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:05.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.128875 20496 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:05.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.130546 20496 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:05.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.131076 20496 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:05.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.133872 20496 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:03:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.138419 20496 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:03:05.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.138670 20496 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:03:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.139021 20496 ts_tablet_manager.cc:616] Registered 0 tablets
03:03:05.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.139271 20496 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:03:05.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.293867 20496 rpc_server.cc:307] RPC server started. Bound to: 127.19.112.67:34891
03:03:05.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.293995 20627 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.19.112.67:34891 every 8 connection(s)
03:03:05.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.297662 20496 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/data/info.pb
03:03:05.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.298483 19905 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 20496
03:03:05.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.298944 19905 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster5551816335887665294/ts-2/wal/instance
03:03:05.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.356295 20628 heartbeater.cc:344] Connected to a master server at 127.19.112.124:35183
03:03:05.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.356825 20628 heartbeater.cc:461] Registering TS with master...
03:03:05.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.358299 20628 heartbeater.cc:507] Master 127.19.112.124:35183 requested a full tablet report, sending...
03:03:05.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.361218 20106 ts_manager.cc:194] Registered new tserver with Master: 792eb096672c4b0a94b91b69e4ec20b6 (127.19.112.67:34891)
03:03:05.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.363206 20106 master_service.cc:502] Signed X509 certificate for tserver {username='slave'} at 127.19.112.67:40143
03:03:05.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.373824 20629 heartbeater.cc:344] Connected to a master server at 127.19.112.125:35049
03:03:05.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.374214 20629 heartbeater.cc:461] Registering TS with master...
03:03:05.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.374960 20629 heartbeater.cc:507] Master 127.19.112.125:35049 requested a full tablet report, sending...
03:03:05.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.377547 20025 ts_manager.cc:194] Registered new tserver with Master: 792eb096672c4b0a94b91b69e4ec20b6 (127.19.112.67:34891)
03:03:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.381949 20632 heartbeater.cc:344] Connected to a master server at 127.19.112.126:46751
03:03:05.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.382231 20632 heartbeater.cc:461] Registering TS with master...
03:03:05.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.382827 20632 heartbeater.cc:507] Master 127.19.112.126:46751 requested a full tablet report, sending...
03:03:05.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.385058 19950 ts_manager.cc:194] Registered new tserver with Master: 792eb096672c4b0a94b91b69e4ec20b6 (127.19.112.67:34891)
03:03:05.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:05.399250 19905 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
03:03:05.411 [DEBUG - main] (MiniKuduCluster.java:195) Response:
03:03:05.412 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
03:03:05.421 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.19.112.126"
port: 46751
}
bound_http_address {
host: "127.19.112.126"
port: 43903
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.19.112.125"
port: 35049
}
bound_http_address {
host: "127.19.112.125"
port: 35761
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.19.112.124"
port: 35183
}
bound_http_address {
host: "127.19.112.124"
port: 46081
}
}
}
03:03:05.424 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:03:05.426 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.19.112.65"
port: 35661
}
bound_http_address {
host: "127.19.112.65"
port: 44043
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.19.112.66"
port: 40557
}
bound_http_address {
host: "127.19.112.66"
port: 39151
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.19.112.67"
port: 34891
}
bound_http_address {
host: "127.19.112.67"
port: 46013
}
}
}
03:03:05.426 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
03:03:05.633 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.base/java.lang.Class.getDeclaredMethod(Class.java:2677)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
03:03:05.699 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
03:03:05.804 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskoRAqye/test-tmp
03:03:05.804 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
03:03:05.805 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskoRAqye/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
03:03:05.808 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
03:03:06.179 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:06.179 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
03:03:06.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:06.182360 20639 mini_oidc.cc:150] Starting JWKS server
03:03:06.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:06.185732 20639 webserver.cc:492] Webserver started at http://127.0.0.1:41191/ using document root <none> and password file <none>
03:03:06.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:06.186290 20639 mini_oidc.cc:186] Starting OIDC Discovery server
03:03:06.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:06.188390 20639 webserver.cc:492] Webserver started at http://127.0.0.1:41457/ using document root <none> and password file <none>
03:03:06.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
03:03:06.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/principal' for realm 'KRBTEST.COM',
03:03:06.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
03:03:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:06 dist-test-slave-sx75 krb5kdc[20651](info): setting up network...
03:03:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
03:03:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:06 dist-test-slave-sx75 krb5kdc[20651](info): set up 2 sockets
03:03:06.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:06 dist-test-slave-sx75 krb5kdc[20651](info): commencing operation
03:03:06.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
03:03:06.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:06.369146 20628 heartbeater.cc:499] Master 127.19.112.124:35183 was elected leader, sending a full tablet report...
03:03:08.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.335477 20639 mini_kdc.cc:121] Time spent starting KDC: real 2.109s user 0.006s sys 0.018s
03:03:08.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
03:03:08.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:03:08.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
03:03:08.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
03:03:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:03:08.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
03:03:08.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
03:03:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:03:08.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
03:03:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
03:03:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/test-user.keytab.
03:03:08.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/test-user.keytab.
03:03:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:08 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975788, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:08.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:03:08.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:03:08Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
03:03:08.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:03:08Z Disabled control of system clock
03:03:08.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.39.254@KRBTEST.COM; defaulting to no policy
03:03:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:08.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.39.254@KRBTEST.COM" created.
03:03:08.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:08.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:08.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:08.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.39.254@KRBTEST.COM; defaulting to no policy
03:03:08.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:08.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.39.254@KRBTEST.COM" created.
03:03:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:08.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:08.575978 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/logs
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/info.pb
03:03:08.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:08.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:08.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:08.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:08.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:08.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:08.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:08.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.254:34561
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.254
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.254
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:03:08.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:08.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.889976 20669 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:08.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.890688 20669 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:08.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.891525 20669 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.930962 20669 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:03:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.931258 20669 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:08.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.931701 20669 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.932044 20669 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.932379 20669 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:08.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.932709 20669 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:08.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.932955 20669 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41191/jwks.json
03:03:08.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:08.972429 20669 master_runner.cc:386] Master server non-default flags:
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.254
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.254:34561
03:03:08.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/info.pb
03:03:08.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.254
03:03:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/logs
03:03:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:08.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:08.974331 20669 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:08.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:08.975828 20669 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:08.990 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.990043 20675 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:10.392799 20674 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 20669
03:03:10.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:10.410332 20669 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.420s user 0.483s sys 0.880s
03:03:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:10.410830 20669 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.421s user 0.484s sys 0.880s
03:03:10.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:08.991274 20676 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:10.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:10.414850 20678 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:10.418623 20677 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1426 milliseconds
03:03:10.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.418838 20669 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:03:10.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.420846 20669 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:10.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.424497 20669 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:10.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.426011 20669 hybrid_clock.cc:648] HybridClock initialized: now 1773975790425978 us; error 31 us; skew 500 ppm
03:03:10.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.427276 20669 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:10.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:10 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975790, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:10.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.433588 20669 init.cc:377] Logged in from keytab as kudu/127.20.39.254@KRBTEST.COM (short username kudu)
03:03:10.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.438998 20669 webserver.cc:492] Webserver started at http://127.20.39.254:37949/ using document root <none> and password file <none>
03:03:10.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.439956 20669 fs_manager.cc:362] Metadata directory not provided
03:03:10.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.440222 20669 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:10.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.440773 20669 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:03:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.445343 20669 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/instance:
03:03:10.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b521a42ad01a4b6aa3e36e623985f78b"
03:03:10.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:10 on dist-test-slave-sx75"
03:03:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.446892 20669 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal/instance:
03:03:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b521a42ad01a4b6aa3e36e623985f78b"
03:03:10.447 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:10 on dist-test-slave-sx75"
03:03:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.455418 20669 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.002s
03:03:10.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.461942 20689 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:10.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.463158 20669 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
03:03:10.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.463564 20669 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b521a42ad01a4b6aa3e36e623985f78b"
03:03:10.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:10 on dist-test-slave-sx75"
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.464579 20669 fs_report.cc:389] FS layout report
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/data
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:10.465 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:10.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.519889 20669 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:10.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.531611 20669 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:10.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.532050 20669 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:10.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.613672 20669 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.254:34561
03:03:10.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.613790 20741 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.254:34561 every 8 connection(s)
03:03:10.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.616671 20669 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/info.pb
03:03:10.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.622730 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 20669
03:03:10.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.622885 20742 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:10.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.623339 20639 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal/instance
03:03:10.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.637594 20742 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } has no permanent_uuid. Determining permanent_uuid...
03:03:10.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.39.253@KRBTEST.COM; defaulting to no policy
03:03:10.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:10.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.39.253@KRBTEST.COM" created.
03:03:10.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:10 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975790, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.254@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:10.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:10.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:10.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.686757 20742 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } has no permanent_uuid. Determining permanent_uuid...
03:03:10.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:10.689409 20692 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:10.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.39.253@KRBTEST.COM; defaulting to no policy
03:03:10.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:10.694090 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.253:34589: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:10.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.39.253@KRBTEST.COM" created.
03:03:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:10.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.723634 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data
03:03:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:10.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/logs
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/info.pb
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.253:34589
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.253
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.253
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:03:10.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:10.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.743597 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } attempt: 1
03:03:10.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:10.747959 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.253:34589: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:10.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.816339 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } attempt: 2
03:03:10.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:10.820938 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.253:34589: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:10.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:10.912454 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } attempt: 3
03:03:10.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:10.918601 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.253:34589: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:11.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:11.062062 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } attempt: 4
03:03:11.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.062369 20751 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:11.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.063143 20751 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:11.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.063961 20751 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:11.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.065987 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.253:34589: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.096395 20751 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:03:11.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.096629 20751 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:11.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.096891 20751 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:11.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.097085 20751 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:11.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.097383 20751 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:11.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.097594 20751 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:11.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.097873 20751 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41191/jwks.json
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:11.135937 20751 master_runner.cc:386] Master server non-default flags:
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.253
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.253:34589
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:11.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/info.pb
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.253
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/logs
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:11.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:11.137156 20751 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:11.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:11.138087 20751 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:11.150 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.150022 20758 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:11.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:11.365435 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } attempt: 5
03:03:11.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.374372 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.253:34589: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:11.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:11.921969 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } attempt: 6
03:03:11.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.945899 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.253:34589: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:12.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:12.551954 20756 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 20751
03:03:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:12.690618 20756 kernel_stack_watchdog.cc:198] Thread 20751 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 400ms:
03:03:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:03:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:03:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:03:12.691 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:03:12.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:12.691198 20751 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.542s user 0.499s sys 0.978s
03:03:12.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:12.691602 20751 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.542s user 0.499s sys 0.978s
03:03:12.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:11.150043 20757 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:12.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:12.692319 20759 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1541 milliseconds
03:03:12.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.693384 20751 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:03:12.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:12.693444 20760 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:12.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.697039 20751 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:12.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.700192 20751 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:12.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.701853 20751 hybrid_clock.cc:648] HybridClock initialized: now 1773975792701772 us; error 87 us; skew 500 ppm
03:03:12.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.703503 20751 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:12.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:12 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975792, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:12.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.711419 20751 init.cc:377] Logged in from keytab as kudu/127.20.39.253@KRBTEST.COM (short username kudu)
03:03:12.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.716325 20751 webserver.cc:492] Webserver started at http://127.20.39.253:40005/ using document root <none> and password file <none>
03:03:12.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.717202 20751 fs_manager.cc:362] Metadata directory not provided
03:03:12.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.717396 20751 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:12.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.717799 20751 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:03:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.722051 20751 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/instance:
03:03:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bca0734326443059a4ce24d90982e7b"
03:03:12.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:12 on dist-test-slave-sx75"
03:03:12.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.723217 20751 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal/instance:
03:03:12.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bca0734326443059a4ce24d90982e7b"
03:03:12.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:12 on dist-test-slave-sx75"
03:03:12.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.730640 20751 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.003s sys 0.004s
03:03:12.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.736128 20772 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:12.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.737185 20751 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.002s
03:03:12.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.737579 20751 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bca0734326443059a4ce24d90982e7b"
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:12 on dist-test-slave-sx75"
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.737929 20751 fs_report.cc:389] FS layout report
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/data
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:12.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:12.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:12.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:12.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:12.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:12.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.793310 20751 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:12.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.798946 20751 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:12.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.799485 20751 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:12.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.879462 20751 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.253:34589
03:03:12.880 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.879618 20830 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.253:34589 every 8 connection(s)
03:03:12.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.882297 20751 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/info.pb
03:03:12.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.888569 20831 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.892014 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 20751
03:03:12.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.892509 20639 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal/instance
03:03:12.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.901473 20831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } has no permanent_uuid. Determining permanent_uuid...
03:03:12.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.39.252@KRBTEST.COM; defaulting to no policy
03:03:12.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:12.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.39.252@KRBTEST.COM" created.
03:03:12.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:12 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975792, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.253@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:12.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:12.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.959949 20831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } has no permanent_uuid. Determining permanent_uuid...
03:03:12.969 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.39.252@KRBTEST.COM; defaulting to no policy
03:03:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:12.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.39.252@KRBTEST.COM" created.
03:03:12.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:12 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975792, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.253@KRBTEST.COM for kudu/127.20.39.253@KRBTEST.COM
03:03:12.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:12.992602 20831 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } has no permanent_uuid. Determining permanent_uuid...
03:03:12.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:12.995282 20774 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:12.999670 20831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:13.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.003999 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/logs
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/info.pb
03:03:13.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:13.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.252:39247
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.252
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.252
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:13.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.006354 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } attempt: 7
03:03:13.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:13 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975790, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.254@KRBTEST.COM for kudu/127.20.39.253@KRBTEST.COM
03:03:13.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.029451 20742 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } has no permanent_uuid. Determining permanent_uuid...
03:03:13.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.035328 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.049193 20831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 1
03:03:13.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.053431 20831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.093811 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 1
03:03:13.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.097968 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.121892 20831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 2
03:03:13.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.125967 20831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.179385 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 2
03:03:13.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.183396 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.217361 20831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 3
03:03:13.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.221820 20831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.268908 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 3
03:03:13.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.273464 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.350601 20843 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:13.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.351181 20843 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:13.352 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.351794 20843 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:13.365 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.365201 20831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 4
03:03:13.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.369125 20831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.384761 20843 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:03:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.384989 20843 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.385248 20843 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.385448 20843 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.385650 20843 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:13.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.385841 20843 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:13.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.385988 20843 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41191/jwks.json
03:03:13.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.413921 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 4
03:03:13.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.417754 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.424043 20843 master_runner.cc:386] Master server non-default flags:
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.252
03:03:13.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.252:39247
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/info.pb
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.252
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/logs
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:13.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.425333 20843 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:13.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.426306 20843 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:13.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.437589 20850 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:13.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.668617 20831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 5
03:03:13.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.679273 20831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:13.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:13.701242 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 5
03:03:13.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.709144 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:14.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.226923 20831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 6
03:03:14.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:14.239058 20831 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:14.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.261847 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 6
03:03:14.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:14.272687 20742 consensus_peers.cc:760] Error getting permanent uuid from config peer 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:14.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.439275 20851 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:14.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:13.449246 20853 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:14.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:14.756615 20852 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1313 milliseconds
03:03:14.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.756757 20843 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:03:14.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.758011 20843 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:14.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.761255 20843 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:14.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.762684 20843 hybrid_clock.cc:648] HybridClock initialized: now 1773975794762640 us; error 65 us; skew 500 ppm
03:03:14.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.763585 20843 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:14.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:14 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975794, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:14.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.770040 20843 init.cc:377] Logged in from keytab as kudu/127.20.39.252@KRBTEST.COM (short username kudu)
03:03:14.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.775621 20843 webserver.cc:492] Webserver started at http://127.20.39.252:34383/ using document root <none> and password file <none>
03:03:14.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.776499 20843 fs_manager.cc:362] Metadata directory not provided
03:03:14.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.776791 20843 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:14.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.777187 20843 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:03:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.781843 20843 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/instance:
03:03:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5ecd815e9f94085b06bcd2fa458c820"
03:03:14.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:14 on dist-test-slave-sx75"
03:03:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.782958 20843 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal/instance:
03:03:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5ecd815e9f94085b06bcd2fa458c820"
03:03:14.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:14 on dist-test-slave-sx75"
03:03:14.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.790263 20843 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.006s sys 0.001s
03:03:14.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.796262 20867 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:14.797 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.797533 20843 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.003s
03:03:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.797878 20843 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5ecd815e9f94085b06bcd2fa458c820"
03:03:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:14 on dist-test-slave-sx75"
03:03:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.798281 20843 fs_report.cc:389] FS layout report
03:03:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:14.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/data
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:14.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:14.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.858438 20843 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:14.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.864431 20843 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:14.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.864842 20843 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.968014 20843 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.252:39247
03:03:14.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.968062 20918 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.252:39247 every 8 connection(s)
03:03:14.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.971935 20843 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/info.pb
03:03:14.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.978271 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 20843
03:03:14.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.978792 20639 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal/instance
03:03:14.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:14.979736 20919 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
03:03:15.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.013844 20919 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } has no permanent_uuid. Determining permanent_uuid...
03:03:15.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:15 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975788, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:15.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:15 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975794, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.252@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:15.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:15 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975788, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.39.253@KRBTEST.COM
03:03:15.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.054915 20919 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } has no permanent_uuid. Determining permanent_uuid...
03:03:15.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:15 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975794, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.252@KRBTEST.COM for kudu/127.20.39.253@KRBTEST.COM
03:03:15.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:15 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975788, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.39.252@KRBTEST.COM
03:03:15.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.080582 20919 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } has no permanent_uuid. Determining permanent_uuid...
03:03:15.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:15 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975794, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.252@KRBTEST.COM for kudu/127.20.39.252@KRBTEST.COM
03:03:15.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.113677 20919 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Bootstrap starting.
03:03:15.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.120142 20919 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Neither blocks nor log segments found. Creating new log.
03:03:15.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.123281 20919 log.cc:826] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Log is configured to *not* fsync() on all Append() calls
03:03:15.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.127653 20919 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: No bootstrap required, opened a new log
03:03:15.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.151952 20919 raft_consensus.cc:359] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.152657 20919 raft_consensus.cc:385] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:15.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.152855 20919 raft_consensus.cc:740] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a5ecd815e9f94085b06bcd2fa458c820, State: Initialized, Role: FOLLOWER
03:03:15.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.153502 20919 consensus_queue.cc:260] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.156840 20930 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:15.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.157507 20930 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: This master's current role is: FOLLOWER
03:03:15.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.158490 20919 sys_catalog.cc:565] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: configured and running, proceeding with master startup.
03:03:15.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:15.187114 20941 catalog_manager.cc:1568] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:03:15.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:15.187475 20941 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:03:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.182262 20843 master_runner.cc:186] Error getting master registration for 127.20.39.254:34561: OK, instance_id { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" instance_seqno: 1773975790591106 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:03:15.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.208487 20843 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:15.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.299706 20831 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 7
03:03:15.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.306317 20742 consensus_peers.cc:770] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } attempt: 7
03:03:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:15 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975790, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.254@KRBTEST.COM for kudu/127.20.39.252@KRBTEST.COM
03:03:15.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:15 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975792, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.253@KRBTEST.COM for kudu/127.20.39.252@KRBTEST.COM
03:03:15.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.365726 20742 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Bootstrap starting.
03:03:15.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:15.370208 20873 tablet.cc:2379] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:03:15.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.372481 20742 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Neither blocks nor log segments found. Creating new log.
03:03:15.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.374765 20742 log.cc:826] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Log is configured to *not* fsync() on all Append() calls
03:03:15.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.375597 20831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Bootstrap starting.
03:03:15.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.380322 20742 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: No bootstrap required, opened a new log
03:03:15.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.390632 20831 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Neither blocks nor log segments found. Creating new log.
03:03:15.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.393216 20831 log.cc:826] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Log is configured to *not* fsync() on all Append() calls
03:03:15.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.399024 20831 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: No bootstrap required, opened a new log
03:03:15.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.411708 20742 raft_consensus.cc:359] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.412602 20742 raft_consensus.cc:385] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:15.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.412943 20742 raft_consensus.cc:740] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b521a42ad01a4b6aa3e36e623985f78b, State: Initialized, Role: FOLLOWER
03:03:15.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.413844 20742 consensus_queue.cc:260] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:15.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.423705 20831 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:15.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.424916 20831 raft_consensus.cc:385] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
03:03:15.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.424944 20947 sys_catalog.cc:455] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:15.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.425405 20831 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2bca0734326443059a4ce24d90982e7b, State: Initialized, Role: FOLLOWER
03:03:15.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.426019 20947 sys_catalog.cc:458] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: This master's current role is: FOLLOWER
03:03:15.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.426407 20742 sys_catalog.cc:565] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: configured and running, proceeding with master startup.
03:03:15.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.426532 20831 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:15.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.430068 20947 raft_consensus.cc:493] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:15.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.430573 20947 raft_consensus.cc:515] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:15.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.432854 20831 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: configured and running, proceeding with master startup.
03:03:15.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.440708 20894 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b521a42ad01a4b6aa3e36e623985f78b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a5ecd815e9f94085b06bcd2fa458c820" is_pre_election: true
03:03:15.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.441880 20894 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b521a42ad01a4b6aa3e36e623985f78b in term 0.
03:03:15.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.447352 20805 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b521a42ad01a4b6aa3e36e623985f78b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bca0734326443059a4ce24d90982e7b" is_pre_election: true
03:03:15.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.448627 20805 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b521a42ad01a4b6aa3e36e623985f78b in term 0.
03:03:15.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.451472 20692 leader_election.cc:304] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2bca0734326443059a4ce24d90982e7b, b521a42ad01a4b6aa3e36e623985f78b; no voters:
03:03:15.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.471642 20947 leader_election.cc:290] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2bca0734326443059a4ce24d90982e7b (127.20.39.253:34589), a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247)
03:03:15.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.472363 20947 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 0 FOLLOWER]: Leader pre-election won for term 1
03:03:15.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.473783 20947 raft_consensus.cc:493] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:03:15.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.474478 20947 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 0 FOLLOWER]: Advancing to term 1
03:03:15.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.493319 20948 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:15.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.500314 20948 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: This master's current role is: FOLLOWER
03:03:15.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:15.505335 20692 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.20.39.253:34589, user_credentials={real_user=slave}} blocked reactor thread for 54040.1us
03:03:15.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.509212 20947 raft_consensus.cc:515] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:15.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.526163 20805 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b521a42ad01a4b6aa3e36e623985f78b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2bca0734326443059a4ce24d90982e7b"
03:03:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.526914 20805 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 0 FOLLOWER]: Advancing to term 1
03:03:15.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.532724 20970 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:03:15.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.536206 20947 leader_election.cc:290] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 1 election: Requested vote from peers 2bca0734326443059a4ce24d90982e7b (127.20.39.253:34589), a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247)
03:03:15.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.537353 20894 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b521a42ad01a4b6aa3e36e623985f78b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a5ecd815e9f94085b06bcd2fa458c820"
03:03:15.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.537976 20894 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 0 FOLLOWER]: Advancing to term 1
03:03:15.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.541577 20805 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b521a42ad01a4b6aa3e36e623985f78b in term 1.
03:03:15.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:15.543149 20695 tablet.cc:2379] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:03:15.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.544802 20692 leader_election.cc:304] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2bca0734326443059a4ce24d90982e7b, b521a42ad01a4b6aa3e36e623985f78b; no voters:
03:03:15.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.546044 20947 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Leader election won for term 1
03:03:15.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.547353 20894 raft_consensus.cc:2468] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b521a42ad01a4b6aa3e36e623985f78b in term 1.
03:03:15.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.547623 20947 raft_consensus.cc:697] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 LEADER]: Becoming Leader. State: Replica: b521a42ad01a4b6aa3e36e623985f78b, State: Running, Role: LEADER
03:03:15.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.550081 20947 consensus_queue.cc:237] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:15.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:15.559930 20778 tablet.cc:2379] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Can't schedule compaction. Clean time has not been advanced past its initial value.
03:03:15.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.562172 20955 sys_catalog.cc:455] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: SysCatalogTable state changed. Reason: New leader b521a42ad01a4b6aa3e36e623985f78b. Latest consensus state: current_term: 1 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:15.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.568670 20955 sys_catalog.cc:458] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: This master's current role is: LEADER
03:03:15.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.541018 20669 master_runner.cc:186] Error getting master registration for 127.20.39.253:34589: OK, instance_id { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" instance_seqno: 1773975792856909 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:03:15.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.575956 20669 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:15.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.577509 20974 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:03:15.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.588631 20974 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:03:15.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.616091 20894 raft_consensus.cc:1275] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 1 FOLLOWER]: Refusing update from remote peer b521a42ad01a4b6aa3e36e623985f78b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:03:15.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.617053 20805 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 1 FOLLOWER]: Refusing update from remote peer b521a42ad01a4b6aa3e36e623985f78b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
03:03:15.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.618021 20947 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [LEADER]: Connected to new peer: Peer: permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:03:15.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.620276 20955 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [LEADER]: Connected to new peer: Peer: permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
03:03:15.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:15.639598 20973 catalog_manager.cc:1568] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
03:03:15.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:15.648510 20973 catalog_manager.cc:883] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
03:03:15.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.676891 20948 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: SysCatalogTable state changed. Reason: New leader b521a42ad01a4b6aa3e36e623985f78b. Latest consensus state: current_term: 1 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.677724 20948 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: This master's current role is: FOLLOWER
03:03:15.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.678619 20930 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b521a42ad01a4b6aa3e36e623985f78b. Latest consensus state: current_term: 1 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:15.681 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.679903 20930 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: This master's current role is: FOLLOWER
03:03:15.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.695873 20979 mvcc.cc:204] Tried to move back new op lower bound from 7266204858816516096 to 7266204858584748032. Current Snapshot: MvccSnapshot[applied={T|T < 7266204858816516096}]
03:03:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.709504 20930 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:15.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.710362 20930 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: This master's current role is: FOLLOWER
03:03:15.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.723167 20955 sys_catalog.cc:455] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:15.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.734364 20948 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:15.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.735185 20948 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: This master's current role is: FOLLOWER
03:03:15.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.731484 20974 catalog_manager.cc:1357] Generated new cluster ID: fd29d4ee9aeb423086fd7ab91f0a3ae4
03:03:15.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.737898 20974 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:03:15.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.732764 20947 sys_catalog.cc:455] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:15.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.746994 20947 sys_catalog.cc:458] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: This master's current role is: LEADER
03:03:15.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.762300 20955 sys_catalog.cc:458] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: This master's current role is: LEADER
03:03:15.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.785713 20974 catalog_manager.cc:1380] Generated new certificate authority record
03:03:15.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.788200 20974 catalog_manager.cc:1514] Loading token signing keys...
03:03:15.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.837144 20974 catalog_manager.cc:6044] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Generated new TSK 0
03:03:15.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:15.838644 20974 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:03:15.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.39.193@KRBTEST.COM; defaulting to no policy
03:03:15.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:15.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.39.193@KRBTEST.COM" created.
03:03:16.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:16.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:16.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:16.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:16.065825 20639 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.20.39.193: real 0.184s user 0.000s sys 0.016s
03:03:16.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.39.193@KRBTEST.COM; defaulting to no policy
03:03:16.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:16.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.39.193@KRBTEST.COM" created.
03:03:16.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:16.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:16.170778 20639 mini_kdc.cc:268] Time spent creating service keytab for HTTP/127.20.39.193: real 0.104s user 0.006s sys 0.009s
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.173957 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/wal
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/data
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/logs
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/data/info.pb
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:16.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:03:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.193:0
03:03:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.39.193
03:03:16.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.193
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.193
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:16.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:16.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.194990 20941 catalog_manager.cc:1269] Loaded cluster ID: fd29d4ee9aeb423086fd7ab91f0a3ae4
03:03:16.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.195324 20941 catalog_manager.cc:1562] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: loading cluster ID for follower catalog manager: success
03:03:16.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.216166 20941 catalog_manager.cc:1584] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: acquiring CA information for follower catalog manager: success
03:03:16.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.222034 20941 catalog_manager.cc:1612] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:03:16.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.655354 20973 catalog_manager.cc:1269] Loaded cluster ID: fd29d4ee9aeb423086fd7ab91f0a3ae4
03:03:16.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.655993 20973 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: loading cluster ID for follower catalog manager: success
03:03:16.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.670194 20973 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: acquiring CA information for follower catalog manager: success
03:03:16.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.696265 20973 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:03:16.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:16.741931 20991 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:16.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:16.742594 20991 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:16.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:16.743678 20991 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:16.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:16.804306 20991 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:16.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:16.806164 20991 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.39.193
03:03:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.875715 20991 tablet_server_runner.cc:78] Tablet server non-default flags:
03:03:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:16.888 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/data
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/wal
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.193
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.193:0
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/data/info.pb
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.193
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.39.193
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/logs
03:03:16.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:16.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:16.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:16.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:03:16.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:16.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:16.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:16.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:16.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:16.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:16.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.890098 20991 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:16.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:16.891713 20991 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:16.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:16.967468 20997 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:19.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:16.987439 20998 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:19.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:16.982071 21000 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:19.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:19.466920 20999 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2484 milliseconds
03:03:19.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.467072 20991 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:03:19.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.468618 20991 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:19.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.472458 20991 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:19.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.474525 20991 hybrid_clock.cc:648] HybridClock initialized: now 1773975799474484 us; error 44 us; skew 500 ppm
03:03:19.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.475873 20991 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:19.483 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:19 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975799, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:19.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.485739 20991 init.cc:377] Logged in from keytab as kudu/127.20.39.193@KRBTEST.COM (short username kudu)
03:03:19.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.492856 20991 webserver.cc:492] Webserver started at http://127.20.39.193:39639/ using document root <none> and password file <none>
03:03:19.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.494089 20991 fs_manager.cc:362] Metadata directory not provided
03:03:19.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.494375 20991 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:19.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.494968 20991 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:03:19.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.508244 20991 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/data/instance:
03:03:19.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4169d8ccfba842878f17e1b90ec3855b"
03:03:19.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:19 on dist-test-slave-sx75"
03:03:19.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.509810 20991 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/wal/instance:
03:03:19.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4169d8ccfba842878f17e1b90ec3855b"
03:03:19.510 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:19 on dist-test-slave-sx75"
03:03:19.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.520476 20991 fs_manager.cc:696] Time spent creating directory manager: real 0.010s user 0.007s sys 0.004s
03:03:19.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.528494 21010 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:19.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.529982 20991 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.005s sys 0.000s
03:03:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.530378 20991 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/wal
03:03:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4169d8ccfba842878f17e1b90ec3855b"
03:03:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:19 on dist-test-slave-sx75"
03:03:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.530843 20991 fs_report.cc:389] FS layout report
03:03:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/wal
03:03:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/wal
03:03:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/data/data
03:03:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:19.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:19.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:19.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:19.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:19.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:19.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:19.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:19.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:19.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:19.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.608436 20991 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:19.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.616438 20991 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:19.617 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.617013 20991 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:19.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.635762 20991 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:03:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.643553 20991 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:03:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.643944 20991 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:03:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.644369 20991 ts_tablet_manager.cc:616] Registered 0 tablets
03:03:19.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.644618 20991 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:03:19.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:19.660032 20482 debug-util.cc:398] Leaking SignalData structure 0x7b08000baaa0 after lost signal to thread 20355
03:03:19.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:19.661437 20482 debug-util.cc:398] Leaking SignalData structure 0x7b08000bac20 after lost signal to thread 20485
03:03:19.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.946905 20991 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.193:35247
03:03:19.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.947063 21125 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.193:35247 every 8 connection(s)
03:03:19.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.951257 20991 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/data/info.pb
03:03:19.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.954767 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 20991
03:03:19.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:19.955266 20639 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-0/wal/instance
03:03:20.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.39.194@KRBTEST.COM; defaulting to no policy
03:03:20.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:20.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.39.194@KRBTEST.COM" created.
03:03:20.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:20 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975799, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.193@KRBTEST.COM for kudu/127.20.39.253@KRBTEST.COM
03:03:20.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:20 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975799, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.193@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:20.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:20 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975799, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.193@KRBTEST.COM for kudu/127.20.39.252@KRBTEST.COM
03:03:20.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.075904 21126 heartbeater.cc:344] Connected to a master server at 127.20.39.253:34589
03:03:20.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.076471 21126 heartbeater.cc:461] Registering TS with master...
03:03:20.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.076663 21133 heartbeater.cc:344] Connected to a master server at 127.20.39.254:34561
03:03:20.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.077042 21133 heartbeater.cc:461] Registering TS with master...
03:03:20.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.077913 21126 heartbeater.cc:507] Master 127.20.39.253:34589 requested a full tablet report, sending...
03:03:20.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.078603 21133 heartbeater.cc:507] Master 127.20.39.254:34561 requested a full tablet report, sending...
03:03:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.081712 20795 ts_manager.cc:194] Registered new tserver with Master: 4169d8ccfba842878f17e1b90ec3855b (127.20.39.193:35247)
03:03:20.082 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.081724 20706 ts_manager.cc:194] Registered new tserver with Master: 4169d8ccfba842878f17e1b90ec3855b (127.20.39.193:35247)
03:03:20.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.085003 20706 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.20.39.193@KRBTEST.COM'} at 127.20.39.193:50443
03:03:20.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.093305 21127 heartbeater.cc:344] Connected to a master server at 127.20.39.252:39247
03:03:20.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.093667 21127 heartbeater.cc:461] Registering TS with master...
03:03:20.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.094353 21127 heartbeater.cc:507] Master 127.20.39.252:39247 requested a full tablet report, sending...
03:03:20.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.097563 20884 ts_manager.cc:194] Registered new tserver with Master: 4169d8ccfba842878f17e1b90ec3855b (127.20.39.193:35247)
03:03:20.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:20.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:20.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:20.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:20.107580 20639 mini_kdc.cc:268] Time spent creating service keytab for kudu/127.20.39.194: real 0.137s user 0.011s sys 0.011s
03:03:20.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.39.194@KRBTEST.COM; defaulting to no policy
03:03:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:20.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.39.194@KRBTEST.COM" created.
03:03:20.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:20.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:20.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.163691 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/wal
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/data
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/logs
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/data/info.pb
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.194:0
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.39.194
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.194
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.194
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:20.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:20.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:20.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:20 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975800, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:20.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.438442 20685 init.cc:295] Successfully reacquired a new kerberos TGT
03:03:20.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:20.525682 21142 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:20.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:20.526136 21142 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:20.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:20.526805 21142 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:20.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:20.561048 21142 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:20.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:20.562350 21142 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.39.194
03:03:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.608716 21142 tablet_server_runner.cc:78] Tablet server non-default flags:
03:03:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:20.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:20.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/data
03:03:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/wal
03:03:20.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.194
03:03:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.194:0
03:03:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:20.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/data/info.pb
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.194
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.39.194
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/logs
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:20.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:03:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.610396 21142 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:20.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:20.611694 21142 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:20.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:20.629030 21149 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:21.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:21.090356 21133 heartbeater.cc:499] Master 127.20.39.254:34561 was elected leader, sending a full tablet report...
03:03:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:22.031091 21148 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 21142
03:03:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:22.295837 21142 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.665s user 0.524s sys 0.914s
03:03:22.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:22.296447 21142 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.666s user 0.524s sys 0.914s
03:03:22.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:20.632334 21150 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:22.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:22.307504 21156 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:22.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.307734 21142 server_base.cc:1061] running on GCE node
03:03:22.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.309278 21142 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:22.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.311892 21142 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:22.313 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.313401 21142 hybrid_clock.cc:648] HybridClock initialized: now 1773975802313285 us; error 113 us; skew 500 ppm
03:03:22.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.314392 21142 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:22.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:22 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975802, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:22.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.320683 21142 init.cc:377] Logged in from keytab as kudu/127.20.39.194@KRBTEST.COM (short username kudu)
03:03:22.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.325948 21142 webserver.cc:492] Webserver started at http://127.20.39.194:40317/ using document root <none> and password file <none>
03:03:22.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.326944 21142 fs_manager.cc:362] Metadata directory not provided
03:03:22.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.327144 21142 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:22.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.327626 21142 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:03:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.333112 21142 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/data/instance:
03:03:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "631f0c8461bb42abad361a00af86e02d"
03:03:22.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:22 on dist-test-slave-sx75"
03:03:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.334354 21142 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/wal/instance:
03:03:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "631f0c8461bb42abad361a00af86e02d"
03:03:22.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:22 on dist-test-slave-sx75"
03:03:22.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.342274 21142 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
03:03:22.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.348042 21162 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:22.349 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.349381 21142 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.349706 21142 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/wal
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "631f0c8461bb42abad361a00af86e02d"
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:22 on dist-test-slave-sx75"
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.350018 21142 fs_report.cc:389] FS layout report
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/wal
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/wal
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/data/data
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:22.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:22.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.405066 21142 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.410837 21142 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:22.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.411337 21142 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:22.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.413776 21142 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:03:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.417986 21142 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:03:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.418179 21142 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:03:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.418378 21142 ts_tablet_manager.cc:616] Registered 0 tablets
03:03:22.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.418526 21142 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:03:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.591989 21142 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.194:39319
03:03:22.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.592164 21277 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.194:39319 every 8 connection(s)
03:03:22.594 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.594379 21142 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/data/info.pb
03:03:22.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.602437 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 21142
03:03:22.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.602980 20639 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-1/wal/instance
03:03:22.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.20.39.195@KRBTEST.COM; defaulting to no policy
03:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:22.659 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.20.39.195@KRBTEST.COM" created.
03:03:22.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:22 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975802, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.194@KRBTEST.COM for kudu/127.20.39.252@KRBTEST.COM
03:03:22.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:22 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975802, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.194@KRBTEST.COM for kudu/127.20.39.253@KRBTEST.COM
03:03:22.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:22 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975802, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.194@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:22.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.685336 21278 heartbeater.cc:344] Connected to a master server at 127.20.39.253:34589
03:03:22.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.685868 21279 heartbeater.cc:344] Connected to a master server at 127.20.39.252:39247
03:03:22.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.686041 21278 heartbeater.cc:461] Registering TS with master...
03:03:22.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.686374 21279 heartbeater.cc:461] Registering TS with master...
03:03:22.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.687970 21278 heartbeater.cc:507] Master 127.20.39.253:34589 requested a full tablet report, sending...
03:03:22.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.688009 21279 heartbeater.cc:507] Master 127.20.39.252:39247 requested a full tablet report, sending...
03:03:22.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.691840 20884 ts_manager.cc:194] Registered new tserver with Master: 631f0c8461bb42abad361a00af86e02d (127.20.39.194:39319)
03:03:22.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.692309 20795 ts_manager.cc:194] Registered new tserver with Master: 631f0c8461bb42abad361a00af86e02d (127.20.39.194:39319)
03:03:22.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.697068 21280 heartbeater.cc:344] Connected to a master server at 127.20.39.254:34561
03:03:22.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.697484 21280 heartbeater.cc:461] Registering TS with master...
03:03:22.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.698799 21280 heartbeater.cc:507] Master 127.20.39.254:34561 requested a full tablet report, sending...
03:03:22.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.701431 20706 ts_manager.cc:194] Registered new tserver with Master: 631f0c8461bb42abad361a00af86e02d (127.20.39.194:39319)
03:03:22.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.703027 20706 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.20.39.194@KRBTEST.COM'} at 127.20.39.194:48579
03:03:22.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:22.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:22.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.20.39.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:22.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:22 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975802, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:22.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.716816 20768 init.cc:295] Successfully reacquired a new kerberos TGT
03:03:22.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.20.39.195@KRBTEST.COM; defaulting to no policy
03:03:22.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:22.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.20.39.195@KRBTEST.COM" created.
03:03:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
03:03:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:22.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.20.39.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab.
03:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:22.760196 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/wal
03:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/data
03:03:22.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/logs
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/data/info.pb
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.195:0
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.39.195
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.195
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.195
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:22.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:22.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:22.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:23.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:23.122627 21292 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:23.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:23.123282 21292 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:23.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:23.124027 21292 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:23.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:23.164346 21292 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:23.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:23.165495 21292 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.20.39.195
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:23.205754 21292 tablet_server_runner.cc:78] Tablet server non-default flags:
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/data
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/wal
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.195
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.195:0
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:23.206 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/data/info.pb
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.195
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.20.39.195
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/logs
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:23.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:23.207554 21292 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:23.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:23.208683 21292 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:23.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:23.226915 21304 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:23.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:23.706508 21280 heartbeater.cc:499] Master 127.20.39.254:34561 was elected leader, sending a full tablet report...
03:03:24.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:24.626791 21302 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ca0 after lost signal to thread 21292
03:03:24.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:24.640573 21302 kernel_stack_watchdog.cc:198] Thread 21292 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 394ms:
03:03:24.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:03:24.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:03:24.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:03:24.641 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:03:24.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:23.227792 21305 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:24.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:24.641745 21292 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.413s user 0.002s sys 0.001s
03:03:24.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:24.642427 21292 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.414s user 0.002s sys 0.001s
03:03:24.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:24.644754 21307 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:24.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.652560 21292 server_base.cc:1061] running on GCE node
03:03:24.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.653709 21292 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:24.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.656406 21292 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:24.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.657850 21292 hybrid_clock.cc:648] HybridClock initialized: now 1773975804657807 us; error 56 us; skew 500 ppm
03:03:24.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.658639 21292 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:24.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:24 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975804, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:24.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.665892 21292 init.cc:377] Logged in from keytab as kudu/127.20.39.195@KRBTEST.COM (short username kudu)
03:03:24.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.671981 21292 webserver.cc:492] Webserver started at http://127.20.39.195:43947/ using document root <none> and password file <none>
03:03:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.672999 21292 fs_manager.cc:362] Metadata directory not provided
03:03:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.673242 21292 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:24.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.673774 21292 server_base.cc:909] This appears to be a new deployment of Kudu; creating new FS layout
03:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.678249 21292 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/data/instance:
03:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c7f0f6f6dcae46e3b8a714ebd1f6b86b"
03:03:24.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:24 on dist-test-slave-sx75"
03:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.679455 21292 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/wal/instance:
03:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c7f0f6f6dcae46e3b8a714ebd1f6b86b"
03:03:24.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:24 on dist-test-slave-sx75"
03:03:24.688 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.687776 21292 fs_manager.cc:696] Time spent creating directory manager: real 0.008s user 0.005s sys 0.002s
03:03:24.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.693835 21319 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:24.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.695226 21292 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.001s
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.695706 21292 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/wal
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "c7f0f6f6dcae46e3b8a714ebd1f6b86b"
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:24 on dist-test-slave-sx75"
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.696139 21292 fs_report.cc:389] FS layout report
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/wal
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/wal
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/data/data
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:24.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:24.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:24.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.759920 21292 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.765841 21292 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:24.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.766309 21292 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:24.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.769306 21292 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
03:03:24.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:24 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975804, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:24.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.774910 21292 ts_tablet_manager.cc:585] Loaded tablet metadata (0 total tablets, 0 live tablets)
03:03:24.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.775159 21292 ts_tablet_manager.cc:531] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
03:03:24.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.775499 21292 ts_tablet_manager.cc:616] Registered 0 tablets
03:03:24.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.775701 21292 ts_tablet_manager.cc:595] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
03:03:24.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:24.775725 20863 init.cc:295] Successfully reacquired a new kerberos TGT
03:03:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.058144 21292 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.195:34001
03:03:25.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.058279 21434 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.195:34001 every 8 connection(s)
03:03:25.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.061015 21292 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/data/info.pb
03:03:25.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.069666 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 21292
03:03:25.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.070161 20639 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/ts-2/wal/instance
03:03:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:25 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975804, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.195@KRBTEST.COM for kudu/127.20.39.253@KRBTEST.COM
03:03:25.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:25 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975804, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.195@KRBTEST.COM for kudu/127.20.39.252@KRBTEST.COM
03:03:25.117 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:25 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975804, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.195@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.138815 21438 heartbeater.cc:344] Connected to a master server at 127.20.39.254:34561
03:03:25.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.139346 21438 heartbeater.cc:461] Registering TS with master...
03:03:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.140880 21435 heartbeater.cc:344] Connected to a master server at 127.20.39.253:34589
03:03:25.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.141275 21435 heartbeater.cc:461] Registering TS with master...
03:03:25.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.142330 21435 heartbeater.cc:507] Master 127.20.39.253:34589 requested a full tablet report, sending...
03:03:25.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.140864 21438 heartbeater.cc:507] Master 127.20.39.254:34561 requested a full tablet report, sending...
03:03:25.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.146746 20706 ts_manager.cc:194] Registered new tserver with Master: c7f0f6f6dcae46e3b8a714ebd1f6b86b (127.20.39.195:34001)
03:03:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.148510 20706 master_service.cc:502] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.20.39.195@KRBTEST.COM'} at 127.20.39.195:60819
03:03:25.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.150329 20795 ts_manager.cc:194] Registered new tserver with Master: c7f0f6f6dcae46e3b8a714ebd1f6b86b (127.20.39.195:34001)
03:03:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.153187 21437 heartbeater.cc:344] Connected to a master server at 127.20.39.252:39247
03:03:25.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.153555 21437 heartbeater.cc:461] Registering TS with master...
03:03:25.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.154798 21437 heartbeater.cc:507] Master 127.20.39.252:39247 requested a full tablet report, sending...
03:03:25.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.157390 20884 ts_manager.cc:194] Registered new tserver with Master: c7f0f6f6dcae46e3b8a714ebd1f6b86b (127.20.39.195:34001)
03:03:25.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.172645 20639 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
03:03:25.184 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:25.185 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
03:03:25.194 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
03:03:25.195 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
03:03:25.198 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.20.39.254"
port: 34561
}
bound_http_address {
host: "127.20.39.254"
port: 37949
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.20.39.253"
port: 34589
}
bound_http_address {
host: "127.20.39.253"
port: 40005
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.20.39.252"
port: 39247
}
bound_http_address {
host: "127.20.39.252"
port: 34383
}
}
}
03:03:25.198 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
03:03:25.200 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.20.39.193"
port: 35247
}
bound_http_address {
host: "127.20.39.193"
port: 39639
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.20.39.194"
port: 39319
}
bound_http_address {
host: "127.20.39.194"
port: 40317
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.20.39.195"
port: 34001
}
bound_http_address {
host: "127.20.39.195"
port: 43947
}
}
}
03:03:25.200 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:03:25.201 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:03:25.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:25 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975805, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:25.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:03:25.242 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:25.245 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc
03:03:25.255 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:03:25.256 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc
03:03:25.257 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:03:25.279 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
03:03:25.281 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:03:25.281 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:03:25.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:25 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975805, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:25.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:03:25.326 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:25.327 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc
03:03:25.329 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:03:25.329 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.39.254:34561
03:03:25.331 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:03:25.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.331884 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 20669
03:03:25.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:25.370 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.39.253:34589
03:03:25.370 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:03:25.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.371270 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 20751
03:03:25.402 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:25.402 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.39.252:39247
03:03:25.403 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:03:25.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.403508 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 20843
03:03:25.431 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:25.431 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.39.254:34561
03:03:25.435 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.437688 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/logs
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/info.pb
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:25.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.254:34561
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.254
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37949
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.254
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:25.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:03:25.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:25.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.708590 21278 heartbeater.cc:646] Failed to heartbeat to 127.20.39.253:34589 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:25.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.770454 21450 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.771023 21450 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:25.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.771665 21450 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.806632 21450 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:03:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.806991 21450 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.807379 21450 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.807601 21450 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:25.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.807824 21450 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:25.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.808022 21450 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:25.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.808171 21450 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41191/jwks.json
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.848786 21450 master_runner.cc:386] Master server non-default flags:
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.254
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.254:34561
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:25.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/info.pb
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.254
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37949
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/logs
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:25.850 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.850085 21450 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:25.851 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:25.851248 21450 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:25.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.862782 21458 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:26.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:26.115267 21126 heartbeater.cc:646] Failed to heartbeat to 127.20.39.253:34589 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:26.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:26.155689 21438 heartbeater.cc:499] Master 127.20.39.254:34561 was elected leader, sending a full tablet report...
03:03:26.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:26.170150 21435 heartbeater.cc:646] Failed to heartbeat to 127.20.39.253:34589 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:27.282 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:25.865001 21459 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:27.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.284965 21461 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:27.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.287869 21460 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1417 milliseconds
03:03:27.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.287976 21450 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:03:27.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.289393 21450 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:27.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.292548 21450 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:27.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.293959 21450 hybrid_clock.cc:648] HybridClock initialized: now 1773975807293917 us; error 40 us; skew 500 ppm
03:03:27.295 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.295131 21450 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:27.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:27 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975807, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:27.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.303963 21450 init.cc:377] Logged in from keytab as kudu/127.20.39.254@KRBTEST.COM (short username kudu)
03:03:27.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.310369 21450 webserver.cc:492] Webserver started at http://127.20.39.254:37949/ using document root <none> and password file <none>
03:03:27.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.311627 21450 fs_manager.cc:362] Metadata directory not provided
03:03:27.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.311923 21450 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:27.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.323256 21450 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
03:03:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.328950 21481 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:27.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.330312 21450 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
03:03:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.330811 21450 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b521a42ad01a4b6aa3e36e623985f78b"
03:03:27.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:10 on dist-test-slave-sx75"
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.333570 21450 fs_report.cc:389] FS layout report
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/data
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:27.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:27.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:27.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:27.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:27.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.401517 21450 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:27.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.411077 21450 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:27.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.411612 21450 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.489174 21450 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.254:34561
03:03:27.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.489307 21532 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.254:34561 every 8 connection(s)
03:03:27.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.491997 21450 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/info.pb
03:03:27.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.500914 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 21450
03:03:27.501 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:27.502 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.39.253:34589
03:03:27.502 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:03:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.504310 21533 sys_catalog.cc:263] Verifying existing consensus state
03:03:27.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.504534 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/logs
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/info.pb
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.253:34589
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.253
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40005
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:27.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.253
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:03:27.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:27.512 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.512229 21533 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Bootstrap starting.
03:03:27.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.557085 21533 log.cc:826] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Log is configured to *not* fsync() on all Append() calls
03:03:27.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.579298 21533 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:03:27.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.580312 21533 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Bootstrap complete.
03:03:27.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.612437 21533 raft_consensus.cc:359] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:27.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.614679 21533 raft_consensus.cc:740] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: b521a42ad01a4b6aa3e36e623985f78b, State: Initialized, Role: FOLLOWER
03:03:27.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.615621 21533 consensus_queue.cc:260] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:27.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.618532 21537 sys_catalog.cc:455] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:27.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.619390 21537 sys_catalog.cc:458] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: This master's current role is: FOLLOWER
03:03:27.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.620496 21533 sys_catalog.cc:565] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: configured and running, proceeding with master startup.
03:03:27.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.663511 21548 catalog_manager.cc:1269] Loaded cluster ID: fd29d4ee9aeb423086fd7ab91f0a3ae4
03:03:27.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.663820 21548 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: loading cluster ID for follower catalog manager: success
03:03:27.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:27 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975807, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.254@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:27.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.672286 21548 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: acquiring CA information for follower catalog manager: success
03:03:27.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.678773 21548 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:03:27.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.694095 21450 master_runner.cc:186] Error getting master registration for 127.20.39.253:34589: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111),
03:03:27.722 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.721819 21450 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:27.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.743532 21280 heartbeater.cc:344] Connected to a master server at 127.20.39.254:34561
03:03:27.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.749760 21498 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "631f0c8461bb42abad361a00af86e02d" instance_seqno: 1773975802555394) as {username='kudu', principal='kudu/127.20.39.194@KRBTEST.COM'} at 127.20.39.194:58931; Asking this server to re-register.
03:03:27.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.751382 21280 heartbeater.cc:461] Registering TS with master...
03:03:27.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.754664 21498 ts_manager.cc:194] Registered new tserver with Master: 631f0c8461bb42abad361a00af86e02d (127.20.39.194:39319)
03:03:27.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.891064 21535 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:27.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.891606 21535 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:27.892 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.892246 21535 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:27.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.929528 21535 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:03:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.929829 21535 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.930255 21535 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:27.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.930603 21535 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.930948 21535 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.931280 21535 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:27.931 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:27.931526 21535 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41191/jwks.json
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.993571 21535 master_runner.cc:386] Master server non-default flags:
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.253
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:27.994 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.253:34589
03:03:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:27.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/info.pb
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.253
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40005
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/logs
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:27.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.995414 21535 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:27.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:27.996896 21535 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:28.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:28.015142 21559 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:28.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:28.213548 21438 heartbeater.cc:344] Connected to a master server at 127.20.39.254:34561
03:03:28.215 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:28.214200 21438 heartbeater.cc:499] Master 127.20.39.254:34561 was elected leader, sending a full tablet report...
03:03:28.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:28.218077 21498 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c7f0f6f6dcae46e3b8a714ebd1f6b86b" instance_seqno: 1773975805020399) as {username='kudu', principal='kudu/127.20.39.195@KRBTEST.COM'} at 127.20.39.195:40201; Asking this server to re-register.
03:03:28.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:28.220575 21438 heartbeater.cc:461] Registering TS with master...
03:03:28.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:28.225200 21498 ts_manager.cc:194] Registered new tserver with Master: c7f0f6f6dcae46e3b8a714ebd1f6b86b (127.20.39.195:34001)
03:03:28.262 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:28.261006 21133 heartbeater.cc:344] Connected to a master server at 127.20.39.254:34561
03:03:28.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:28.264552 21498 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "4169d8ccfba842878f17e1b90ec3855b" instance_seqno: 1773975799860149) as {username='kudu', principal='kudu/127.20.39.193@KRBTEST.COM'} at 127.20.39.193:57119; Asking this server to re-register.
03:03:28.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:28.266683 21133 heartbeater.cc:461] Registering TS with master...
03:03:28.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:28.270241 21498 ts_manager.cc:194] Registered new tserver with Master: 4169d8ccfba842878f17e1b90ec3855b (127.20.39.193:35247)
03:03:29.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.329006 21574 raft_consensus.cc:493] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:29.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.330559 21574 raft_consensus.cc:515] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:29.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:29.388588 21484 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:29.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.393220 21574 leader_election.cc:290] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2bca0734326443059a4ce24d90982e7b (127.20.39.253:34589), a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247)
03:03:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:29.402247 21483 leader_election.cc:336] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247): Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:29.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:29.407653 21484 leader_election.cc:336] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 2bca0734326443059a4ce24d90982e7b (127.20.39.253:34589): Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.409196 21484 leader_election.cc:304] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b521a42ad01a4b6aa3e36e623985f78b; no voters: 2bca0734326443059a4ce24d90982e7b, a5ecd815e9f94085b06bcd2fa458c820
03:03:29.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.418569 21574 raft_consensus.cc:2749] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
03:03:29.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:29 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975809, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:29.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.492115 21006 init.cc:295] Successfully reacquired a new kerberos TGT
03:03:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:29.416378 21558 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 21535
03:03:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:29.610937 21558 kernel_stack_watchdog.cc:198] Thread 21535 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 395ms:
03:03:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:03:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:03:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:03:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:03:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:29.611065 21535 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.594s user 0.571s sys 0.940s
03:03:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:29.611464 21535 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.595s user 0.571s sys 0.940s
03:03:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:28.019101 21560 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.621137 21535 server_base.cc:1061] running on GCE node
03:03:29.621 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:29.621634 21579 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:29.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.623395 21535 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:29.627 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.626641 21535 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.628154 21535 hybrid_clock.cc:648] HybridClock initialized: now 1773975809628033 us; error 121 us; skew 500 ppm
03:03:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.629237 21535 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:29.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:29 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975809, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:29.637 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.637151 21535 init.cc:377] Logged in from keytab as kudu/127.20.39.253@KRBTEST.COM (short username kudu)
03:03:29.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.642855 21535 webserver.cc:492] Webserver started at http://127.20.39.253:40005/ using document root <none> and password file <none>
03:03:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.644013 21535 fs_manager.cc:362] Metadata directory not provided
03:03:29.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.644294 21535 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:29.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.655205 21535 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.006s sys 0.001s
03:03:29.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.660885 21586 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:29.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.662213 21535 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
03:03:29.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.662546 21535 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:29.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bca0734326443059a4ce24d90982e7b"
03:03:29.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:12 on dist-test-slave-sx75"
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.664640 21535 fs_report.cc:389] FS layout report
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/data
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:29.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:29.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:29.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:29.732 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.732018 21535 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:29.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.738348 21535 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:29.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.738937 21535 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.834534 21535 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.253:34589
03:03:29.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.834654 21640 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.253:34589 every 8 connection(s)
03:03:29.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.838351 21535 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/info.pb
03:03:29.841 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:29.842 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.39.252:39247
03:03:29.842 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:03:29.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.840412 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 21535
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.844355 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/logs
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/info.pb
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:29.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.252:39247
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.252
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34383
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.252
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:03:29.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:29.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.860513 21641 sys_catalog.cc:263] Verifying existing consensus state
03:03:29.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.870263 21641 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Bootstrap starting.
03:03:29.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.939782 21641 log.cc:826] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Log is configured to *not* fsync() on all Append() calls
03:03:29.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.967618 21641 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:03:29.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:29.969926 21641 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Bootstrap complete.
03:03:30.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.011590 21641 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:30.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.016808 21641 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2bca0734326443059a4ce24d90982e7b, State: Initialized, Role: FOLLOWER
03:03:30.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.018253 21641 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:30.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.027377 21645 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.031709 21645 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: This master's current role is: FOLLOWER
03:03:30.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.031347 21641 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: configured and running, proceeding with master startup.
03:03:30.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.105939 21656 catalog_manager.cc:1269] Loaded cluster ID: fd29d4ee9aeb423086fd7ab91f0a3ae4
03:03:30.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.106595 21656 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: loading cluster ID for follower catalog manager: success
03:03:30.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:30 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975809, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.253@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.119336 21656 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: acquiring CA information for follower catalog manager: success
03:03:30.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.125030 21656 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:03:30.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.181300 21535 master_runner.cc:186] Error getting master registration for 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111),
03:03:30.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.220147 21535 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:30.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.223807 21126 heartbeater.cc:344] Connected to a master server at 127.20.39.253:34589
03:03:30.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.230141 21606 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "4169d8ccfba842878f17e1b90ec3855b" instance_seqno: 1773975799860149) as {username='kudu', principal='kudu/127.20.39.193@KRBTEST.COM'} at 127.20.39.193:57919; Asking this server to re-register.
03:03:30.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.232308 21126 heartbeater.cc:461] Registering TS with master...
03:03:30.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.236303 21605 ts_manager.cc:194] Registered new tserver with Master: 4169d8ccfba842878f17e1b90ec3855b (127.20.39.193:35247)
03:03:30.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.319739 21435 heartbeater.cc:344] Connected to a master server at 127.20.39.253:34589
03:03:30.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.322336 21605 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c7f0f6f6dcae46e3b8a714ebd1f6b86b" instance_seqno: 1773975805020399) as {username='kudu', principal='kudu/127.20.39.195@KRBTEST.COM'} at 127.20.39.195:49219; Asking this server to re-register.
03:03:30.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.323807 21435 heartbeater.cc:461] Registering TS with master...
03:03:30.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.326900 21605 ts_manager.cc:194] Registered new tserver with Master: c7f0f6f6dcae46e3b8a714ebd1f6b86b (127.20.39.195:34001)
03:03:30.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.374825 21643 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:30.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.375638 21643 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:30.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.376832 21643 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:30.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.434497 21643 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:03:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.434827 21643 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.435232 21643 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:30.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.435591 21643 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:30.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.435913 21643 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:30.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.436219 21643 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:30.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.436544 21643 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41191/jwks.json
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.477743 21643 master_runner.cc:386] Master server non-default flags:
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.252
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.252:39247
03:03:30.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/info.pb
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.252
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34383
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/logs
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:30.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.479565 21643 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:30.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.481000 21643 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:30.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.560883 21671 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:30.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.796746 21278 heartbeater.cc:344] Connected to a master server at 127.20.39.253:34589
03:03:30.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.801681 21605 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "631f0c8461bb42abad361a00af86e02d" instance_seqno: 1773975802555394) as {username='kudu', principal='kudu/127.20.39.194@KRBTEST.COM'} at 127.20.39.194:37283; Asking this server to re-register.
03:03:30.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.805354 21278 heartbeater.cc:461] Registering TS with master...
03:03:30.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:30.812664 21605 ts_manager.cc:194] Registered new tserver with Master: 631f0c8461bb42abad361a00af86e02d (127.20.39.194:39319)
03:03:31.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.273017 21678 raft_consensus.cc:493] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:31.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.274129 21678 raft_consensus.cc:515] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:31.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.290804 21678 leader_election.cc:290] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 2bca0734326443059a4ce24d90982e7b (127.20.39.253:34589), a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247)
03:03:31.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:31.321820 21483 leader_election.cc:336] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247): Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:31.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.324530 21616 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b521a42ad01a4b6aa3e36e623985f78b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2bca0734326443059a4ce24d90982e7b" is_pre_election: true
03:03:31.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.325465 21616 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b521a42ad01a4b6aa3e36e623985f78b in term 1.
03:03:31.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.327069 21484 leader_election.cc:304] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2bca0734326443059a4ce24d90982e7b, b521a42ad01a4b6aa3e36e623985f78b; no voters: a5ecd815e9f94085b06bcd2fa458c820
03:03:31.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.328086 21678 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Leader pre-election won for term 2
03:03:31.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.328424 21678 raft_consensus.cc:493] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:03:31.335 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.328727 21678 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 1 FOLLOWER]: Advancing to term 2
03:03:31.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.338443 21678 raft_consensus.cc:515] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:31.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.341635 21616 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b521a42ad01a4b6aa3e36e623985f78b" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "2bca0734326443059a4ce24d90982e7b"
03:03:31.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.342185 21616 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 1 FOLLOWER]: Advancing to term 2
03:03:31.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:31.355538 21483 leader_election.cc:336] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247): Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:31.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.357693 21616 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b521a42ad01a4b6aa3e36e623985f78b in term 2.
03:03:31.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.358295 21678 leader_election.cc:290] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 2 election: Requested vote from peers 2bca0734326443059a4ce24d90982e7b (127.20.39.253:34589), a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247)
03:03:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.360363 21484 leader_election.cc:304] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2bca0734326443059a4ce24d90982e7b, b521a42ad01a4b6aa3e36e623985f78b; no voters: a5ecd815e9f94085b06bcd2fa458c820
03:03:31.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.361580 21678 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Leader election won for term 2
03:03:31.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.366261 21678 raft_consensus.cc:697] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 LEADER]: Becoming Leader. State: Replica: b521a42ad01a4b6aa3e36e623985f78b, State: Running, Role: LEADER
03:03:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.370215 21678 consensus_queue.cc:237] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:31.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.386746 21683 sys_catalog.cc:455] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: SysCatalogTable state changed. Reason: New leader b521a42ad01a4b6aa3e36e623985f78b. Latest consensus state: current_term: 2 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:31.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.387938 21683 sys_catalog.cc:458] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: This master's current role is: LEADER
03:03:31.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.396811 21685 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:03:31.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.405613 21685 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:03:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.409847 21685 catalog_manager.cc:1269] Loaded cluster ID: fd29d4ee9aeb423086fd7ab91f0a3ae4
03:03:31.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.410334 21685 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:03:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.415889 21685 catalog_manager.cc:1514] Loading token signing keys...
03:03:31.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.420166 21685 catalog_manager.cc:6055] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Loaded TSK: 0
03:03:31.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.421690 21685 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:03:31.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.806630 21616 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 2 FOLLOWER]: Refusing update from remote peer b521a42ad01a4b6aa3e36e623985f78b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
03:03:31.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.808777 21683 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [LEADER]: Connected to new peer: Peer: permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
03:03:31.857 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.855890 21688 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: SysCatalogTable state changed. Reason: New leader b521a42ad01a4b6aa3e36e623985f78b. Latest consensus state: current_term: 2 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:31.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.857620 21688 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: This master's current role is: FOLLOWER
03:03:31.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.871847 21688 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:31.881 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.880766 21688 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: This master's current role is: FOLLOWER
03:03:31.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.899183 21678 sys_catalog.cc:455] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:31.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:31.899940 21483 consensus_peers.cc:597] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b -> Peer a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247): Couldn't send request to peer a5ecd815e9f94085b06bcd2fa458c820. Status: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:03:31.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:31.900274 21678 sys_catalog.cc:458] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: This master's current role is: LEADER
03:03:32.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:31.955453 21669 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 21643
03:03:32.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:32.280791 21643 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.694s user 0.537s sys 0.832s
03:03:32.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:30.604087 21672 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:32.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:32.281276 21643 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.695s user 0.537s sys 0.832s
03:03:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:32.290527 21696 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:32.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.290617 21643 server_base.cc:1061] running on GCE node
03:03:32.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.292227 21643 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:32.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.296195 21643 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:32.297 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.297672 21643 hybrid_clock.cc:648] HybridClock initialized: now 1773975812297620 us; error 65 us; skew 500 ppm
03:03:32.299 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.298825 21643 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:32.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:32 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975812, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:32.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.307847 21643 init.cc:377] Logged in from keytab as kudu/127.20.39.252@KRBTEST.COM (short username kudu)
03:03:32.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.317196 21643 webserver.cc:492] Webserver started at http://127.20.39.252:34383/ using document root <none> and password file <none>
03:03:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.318733 21643 fs_manager.cc:362] Metadata directory not provided
03:03:32.319 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.319272 21643 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:32.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:32 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975812, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:32.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.325763 21158 init.cc:295] Successfully reacquired a new kerberos TGT
03:03:32.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.330543 21643 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.002s sys 0.004s
03:03:32.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.337714 21703 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:32.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.339499 21643 fs_manager.cc:730] Time spent opening block manager: real 0.006s user 0.006s sys 0.000s
03:03:32.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.339936 21643 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:32.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5ecd815e9f94085b06bcd2fa458c820"
03:03:32.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:14 on dist-test-slave-sx75"
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.342779 21643 fs_report.cc:389] FS layout report
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/data
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:32.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:32.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.418913 21643 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:32.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.433766 21643 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:32.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.434440 21643 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:32.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.547253 21643 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.252:39247
03:03:32.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.547313 21756 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.252:39247 every 8 connection(s)
03:03:32.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.551481 21643 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/info.pb
03:03:32.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.554069 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 21643
03:03:32.555 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:32.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.570510 21758 sys_catalog.cc:263] Verifying existing consensus state
03:03:32.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.584242 21758 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Bootstrap starting.
03:03:32.627 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.39.254:34561(127.20.39.254:34561)] connecting to peer
03:03:32.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.636301 21758 log.cc:826] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Log is configured to *not* fsync() on all Append() calls
03:03:32.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.675058 21758 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:03:32.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.675961 21758 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Bootstrap complete.
03:03:32.709 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.39.253:34589(127.20.39.253:34589)] connecting to peer
03:03:32.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.707664 21758 raft_consensus.cc:359] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:32.711 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.39.252:39247(127.20.39.252:39247)] connecting to peer
03:03:32.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.711031 21758 raft_consensus.cc:740] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: a5ecd815e9f94085b06bcd2fa458c820, State: Initialized, Role: FOLLOWER
03:03:32.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.712620 21758 consensus_queue.cc:260] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:32.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.725909 21765 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:32.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.727460 21765 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: This master's current role is: FOLLOWER
03:03:32.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.735891 21758 sys_catalog.cc:565] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: configured and running, proceeding with master startup.
03:03:32.736 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.20.39.254:34561(127.20.39.254:34561)] Successfully connected to peer
03:03:32.737 [DEBUG - kudu-nio-2] (Connection.java:757) [peer master-127.20.39.252:39247(127.20.39.252:39247)] Successfully connected to peer
03:03:32.757 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.20.39.253:34589(127.20.39.253:34589)] Successfully connected to peer
03:03:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.780654 21280 heartbeater.cc:499] Master 127.20.39.254:34561 was elected leader, sending a full tablet report...
03:03:32.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:32 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975807, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.254@KRBTEST.COM for kudu/127.20.39.252@KRBTEST.COM
03:03:32.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.852619 21450 master_runner.cc:186] Error getting master registration for 127.20.39.252:39247: OK, instance_id { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" instance_seqno: 1773975812507935 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
03:03:32.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.853324 21450 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:32.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.868470 21732 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 1 FOLLOWER]: Advancing to term 2
03:03:32.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:32 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975812, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.194@KRBTEST.COM for kudu/127.20.39.252@KRBTEST.COM
03:03:32.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.889707 21279 heartbeater.cc:344] Connected to a master server at 127.20.39.252:39247
03:03:32.933 [WARN - kudu-nio-0] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
03:03:32.966 [DEBUG - kudu-nio-2] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.39.252
03:03:32.971 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.39.253
03:03:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.964881 21765 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b521a42ad01a4b6aa3e36e623985f78b. Latest consensus state: current_term: 2 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:32.971 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism GSSAPI chosen for peer 127.20.39.254
03:03:32.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.971273 21765 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: This master's current role is: FOLLOWER
03:03:32.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.987874 21783 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:32.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:32.995328 21783 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: This master's current role is: FOLLOWER
03:03:33.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.097817 21786 catalog_manager.cc:1269] Loaded cluster ID: fd29d4ee9aeb423086fd7ab91f0a3ae4
03:03:33.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.098394 21786 catalog_manager.cc:1562] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: loading cluster ID for follower catalog manager: success
03:03:33.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:33 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975812, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.252@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:33.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.156112 21786 catalog_manager.cc:1584] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: acquiring CA information for follower catalog manager: success
03:03:33.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.199132 21786 catalog_manager.cc:1612] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:03:33.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.256522 21438 heartbeater.cc:499] Master 127.20.39.254:34561 was elected leader, sending a full tablet report...
03:03:33.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.272727 21127 heartbeater.cc:344] Connected to a master server at 127.20.39.252:39247
03:03:33.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.278993 21720 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "4169d8ccfba842878f17e1b90ec3855b" instance_seqno: 1773975799860149) as {username='kudu', principal='kudu/127.20.39.193@KRBTEST.COM'} at 127.20.39.193:46145; Asking this server to re-register.
03:03:33.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.281415 21127 heartbeater.cc:461] Registering TS with master...
03:03:33.286 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.285813 21719 ts_manager.cc:194] Registered new tserver with Master: 4169d8ccfba842878f17e1b90ec3855b (127.20.39.193:35247)
03:03:33.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.308876 21133 heartbeater.cc:499] Master 127.20.39.254:34561 was elected leader, sending a full tablet report...
03:03:33.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.369823 21683 sys_catalog.cc:455] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:33.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.370937 21683 sys_catalog.cc:458] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: This master's current role is: LEADER
03:03:33.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.375476 21437 heartbeater.cc:344] Connected to a master server at 127.20.39.252:39247
03:03:33.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.378430 21719 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c7f0f6f6dcae46e3b8a714ebd1f6b86b" instance_seqno: 1773975805020399) as {username='kudu', principal='kudu/127.20.39.195@KRBTEST.COM'} at 127.20.39.195:55169; Asking this server to re-register.
03:03:33.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.380148 21437 heartbeater.cc:461] Registering TS with master...
03:03:33.384 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.383471 21719 ts_manager.cc:194] Registered new tserver with Master: c7f0f6f6dcae46e3b8a714ebd1f6b86b (127.20.39.195:34001)
03:03:33.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:33.427322 21637 debug-util.cc:398] Leaking SignalData structure 0x7b08000b15c0 after lost signal to thread 21554
03:03:33.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:33 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773975805, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.39.253@KRBTEST.COM
03:03:33.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:33 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773975805, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.39.252@KRBTEST.COM
03:03:33.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:33 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (1 etypes {17}) 127.0.0.1: ISSUE: authtime 1773975805, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:33.637 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x6388c6cb, L:/127.0.0.1:43260 - R:/127.20.39.254:34561] using SASL/GSSAPI
03:03:33.637 [DEBUG - kudu-nio-2] (Negotiator.java:919) Authenticated connection [id: 0x4e6069f1, L:/127.0.0.1:43862 - R:/127.20.39.252:39247] using SASL/GSSAPI
03:03:33.639 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x04b5fffc, L:/127.0.0.1:48738 - R:/127.20.39.253:34589] using SASL/GSSAPI
03:03:33.765 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
03:03:33.767 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=1208), Traces: [0ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.39.254:34561, [142ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.39.253:34589, [144ms] Sub RPC ConnectToMaster: sending RPC to server master-127.20.39.252:39247, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.20.39.252:39247: OK, [1122ms] Sub RPC ConnectToMaster: received response from server master-127.20.39.254:34561: OK, [1130ms] Sub RPC ConnectToMaster: received response from server master-127.20.39.253:34589: OK, deferred=null) after lookup
03:03:33.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.902848 21719 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "631f0c8461bb42abad361a00af86e02d" instance_seqno: 1773975802555394) as {username='kudu', principal='kudu/127.20.39.194@KRBTEST.COM'} at 127.20.39.194:37525; Asking this server to re-register.
03:03:33.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.904644 21279 heartbeater.cc:461] Registering TS with master...
03:03:33.907 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:33.907449 21719 ts_manager.cc:194] Registered new tserver with Master: 631f0c8461bb42abad361a00af86e02d (127.20.39.194:39319)
03:03:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:34 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975814, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:34.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:34.671486 21315 init.cc:295] Successfully reacquired a new kerberos TGT
03:03:34.796 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
03:03:34.797 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
03:03:34.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:34 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975814, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:34.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
03:03:34.851 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:34.851 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc
03:03:34.853 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
03:03:34.853 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.39.254:34561
03:03:34.853 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
03:03:34.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:34.854413 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 21450
03:03:35.003 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:35.007 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.39.253:34589
03:03:35.009 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
03:03:35.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:35.010807 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 21535
03:03:35.037 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.20.39.254:34561(127.20.39.254:34561)] handling channelInactive
03:03:35.039 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.20.39.254:34561(127.20.39.254:34561)] cleaning up while in state READY due to: connection closed
03:03:35.183 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:35.183 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.20.39.252:39247
03:03:35.183 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.20.39.253:34589(127.20.39.253:34589)] handling channelInactive
03:03:35.183 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
03:03:35.184 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.20.39.253:34589(127.20.39.253:34589)] cleaning up while in state READY due to: connection closed
03:03:35.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:35.184280 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 21643
03:03:35.216 [DEBUG - kudu-nio-2] (Connection.java:239) [peer master-127.20.39.252:39247(127.20.39.252:39247)] handling channelInactive
03:03:35.216 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:35.216 [DEBUG - kudu-nio-2] (Connection.java:709) [peer master-127.20.39.252:39247(127.20.39.252:39247)] cleaning up while in state READY due to: connection closed
03:03:35.216 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.39.254:34561
03:03:35.217 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:35.218613 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/logs
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/info.pb
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.254:34561
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.254
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37949
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.254
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:35.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:03:35.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.555943 21800 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.556722 21800 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:35.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.557605 21800 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.597824 21800 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:03:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.598166 21800 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:35.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.598644 21800 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:35.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.599000 21800 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:35.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.599357 21800 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:35.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.599691 21800 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:35.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.599953 21800 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41191/jwks.json
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:35.642699 21800 master_runner.cc:386] Master server non-default flags:
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:35.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.254
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.254:34561
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/info.pb
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.254
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37949
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/logs
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:35.644 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:35.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:35.644544 21800 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:35.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:35.645902 21800 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:35.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.659776 21808 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:37.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:37.063647 21807 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 21800
03:03:37.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:37.395867 21800 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.735s user 0.529s sys 1.029s
03:03:37.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:35.662322 21809 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:37.398798 21810 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1735 milliseconds
03:03:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:37.398912 21800 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.738s user 0.530s sys 1.029s
03:03:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:37.398566 21811 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:37.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.399242 21800 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:03:37.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.406363 21800 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:37.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.409106 21800 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:37.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.410583 21800 hybrid_clock.cc:648] HybridClock initialized: now 1773975817410547 us; error 38 us; skew 500 ppm
03:03:37.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.411792 21800 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:37.417 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:37 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975817, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:37.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.417888 21800 init.cc:377] Logged in from keytab as kudu/127.20.39.254@KRBTEST.COM (short username kudu)
03:03:37.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.423484 21800 webserver.cc:492] Webserver started at http://127.20.39.254:37949/ using document root <none> and password file <none>
03:03:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.424335 21800 fs_manager.cc:362] Metadata directory not provided
03:03:37.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.424541 21800 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:37.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.432884 21800 fs_manager.cc:714] Time spent opening directory manager: real 0.005s user 0.000s sys 0.005s
03:03:37.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.437453 21827 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:37.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.438649 21800 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.001s sys 0.001s
03:03:37.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.439008 21800 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:37.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b521a42ad01a4b6aa3e36e623985f78b"
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:10 on dist-test-slave-sx75"
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.440902 21800 fs_report.cc:389] FS layout report
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/wal
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/data
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:37.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:37.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.498559 21800 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:37.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.507592 21800 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:37.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.508229 21800 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.586002 21800 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.254:34561
03:03:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.586083 21878 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.254:34561 every 8 connection(s)
03:03:37.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.589130 21800 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-0/data/info.pb
03:03:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.597453 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 21800
03:03:37.598 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:37.598 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.39.253:34589
03:03:37.599 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
03:03:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.601714 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/logs
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/info.pb
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.253:34589
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.253
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40005
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.253
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:37.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.602032 21879 sys_catalog.cc:263] Verifying existing consensus state
03:03:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.609364 21879 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Bootstrap starting.
03:03:37.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.646261 21879 log.cc:826] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Log is configured to *not* fsync() on all Append() calls
03:03:37.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.666385 21879 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:03:37.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.667237 21879 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Bootstrap complete.
03:03:37.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.692111 21879 raft_consensus.cc:359] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:37.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.694371 21879 raft_consensus.cc:740] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: b521a42ad01a4b6aa3e36e623985f78b, State: Initialized, Role: FOLLOWER
03:03:37.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.695220 21879 consensus_queue.cc:260] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:37.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.698124 21883 sys_catalog.cc:455] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:37.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.698993 21883 sys_catalog.cc:458] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: This master's current role is: FOLLOWER
03:03:37.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.700050 21879 sys_catalog.cc:565] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: configured and running, proceeding with master startup.
03:03:37.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.736462 21894 catalog_manager.cc:1269] Loaded cluster ID: fd29d4ee9aeb423086fd7ab91f0a3ae4
03:03:37.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.736786 21894 catalog_manager.cc:1562] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: loading cluster ID for follower catalog manager: success
03:03:37.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.745213 21894 catalog_manager.cc:1584] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: acquiring CA information for follower catalog manager: success
03:03:37.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.751204 21894 catalog_manager.cc:1612] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:03:37.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:37 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975817, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.254@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.774674 21800 master_runner.cc:186] Error getting master registration for 127.20.39.253:34589: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111),
03:03:37.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.802246 21800 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:37.846 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.846369 21280 heartbeater.cc:344] Connected to a master server at 127.20.39.254:34561
03:03:37.853 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.852984 21844 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "631f0c8461bb42abad361a00af86e02d" instance_seqno: 1773975802555394) as {username='kudu', principal='kudu/127.20.39.194@KRBTEST.COM'} at 127.20.39.194:52583; Asking this server to re-register.
03:03:37.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.854748 21280 heartbeater.cc:461] Registering TS with master...
03:03:37.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:37.858246 21844 ts_manager.cc:194] Registered new tserver with Master: 631f0c8461bb42abad361a00af86e02d (127.20.39.194:39319)
03:03:37.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:37.978845 21881 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:37.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:37.979522 21881 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:37.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:37.980834 21881 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:38.019349 21881 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:03:38.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:38.019645 21881 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:38.020061 21881 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:38.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:38.020398 21881 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:38.020749 21881 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:38.021056 21881 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:38.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:38.021220 21881 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41191/jwks.json
03:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:38.062280 21881 master_runner.cc:386] Master server non-default flags:
03:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.253
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.253:34589
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/info.pb
03:03:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.253
03:03:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=40005
03:03:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/logs
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:38.064002 21881 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:38.065362 21881 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:38.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:38.077272 21905 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:38.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:38.331306 21438 heartbeater.cc:344] Connected to a master server at 127.20.39.254:34561
03:03:38.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:38.334105 21844 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c7f0f6f6dcae46e3b8a714ebd1f6b86b" instance_seqno: 1773975805020399) as {username='kudu', principal='kudu/127.20.39.195@KRBTEST.COM'} at 127.20.39.195:32827; Asking this server to re-register.
03:03:38.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:38.335744 21438 heartbeater.cc:461] Registering TS with master...
03:03:38.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:38.338972 21844 ts_manager.cc:194] Registered new tserver with Master: c7f0f6f6dcae46e3b8a714ebd1f6b86b (127.20.39.195:34001)
03:03:38.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:38.353245 21133 heartbeater.cc:344] Connected to a master server at 127.20.39.254:34561
03:03:38.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:38.356925 21844 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "4169d8ccfba842878f17e1b90ec3855b" instance_seqno: 1773975799860149) as {username='kudu', principal='kudu/127.20.39.193@KRBTEST.COM'} at 127.20.39.193:42251; Asking this server to re-register.
03:03:38.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:38.358259 21133 heartbeater.cc:461] Registering TS with master...
03:03:38.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:38.361971 21844 ts_manager.cc:194] Registered new tserver with Master: 4169d8ccfba842878f17e1b90ec3855b (127.20.39.193:35247)
03:03:39.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:39.113775 21917 raft_consensus.cc:493] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:39.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:39.114610 21917 raft_consensus.cc:515] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:39.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:39.133013 21830 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:39.155 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:39.154822 21917 leader_election.cc:290] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 2bca0734326443059a4ce24d90982e7b (127.20.39.253:34589), a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247)
03:03:39.168 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:39.167829 21830 leader_election.cc:336] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 2bca0734326443059a4ce24d90982e7b (127.20.39.253:34589): Network error: Client connection negotiation failed: client connection to 127.20.39.253:34589: connect: Connection refused (error 111)
03:03:39.178 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:39.177979 21829 leader_election.cc:336] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247): Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:39.178488 21829 leader_election.cc:304] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b521a42ad01a4b6aa3e36e623985f78b; no voters: 2bca0734326443059a4ce24d90982e7b, a5ecd815e9f94085b06bcd2fa458c820
03:03:39.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:39.179373 21917 raft_consensus.cc:2749] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
03:03:39.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:39 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975819, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:39.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:39.496376 21006 init.cc:295] Successfully reacquired a new kerberos TGT
03:03:40.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:39.483170 21904 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 21881
03:03:40.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:40.303946 21904 kernel_stack_watchdog.cc:198] Thread 21881 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:642 for 401ms:
03:03:40.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
03:03:40.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
03:03:40.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
03:03:40.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
03:03:40.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:38.079353 21907 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:40.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:40.314069 21881 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.235s user 0.707s sys 1.125s
03:03:40.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:40.314484 21881 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.236s user 0.707s sys 1.125s
03:03:40.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:40.315435 21908 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2235 milliseconds
03:03:40.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:40.316635 21909 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:40.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.316727 21881 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:03:40.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.318229 21881 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:40.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.321605 21881 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:40.323 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.323035 21881 hybrid_clock.cc:648] HybridClock initialized: now 1773975820323006 us; error 35 us; skew 500 ppm
03:03:40.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.323920 21881 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:40.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:40 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975820, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:40.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.331610 21881 init.cc:377] Logged in from keytab as kudu/127.20.39.253@KRBTEST.COM (short username kudu)
03:03:40.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.338168 21881 webserver.cc:492] Webserver started at http://127.20.39.253:40005/ using document root <none> and password file <none>
03:03:40.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.339753 21881 fs_manager.cc:362] Metadata directory not provided
03:03:40.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.340094 21881 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:40.351 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.351241 21881 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.001s
03:03:40.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.357085 21933 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:40.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.358455 21881 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
03:03:40.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.358892 21881 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:40.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2bca0734326443059a4ce24d90982e7b"
03:03:40.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:12 on dist-test-slave-sx75"
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.361721 21881 fs_report.cc:389] FS layout report
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/wal
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/data
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:40.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:40.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:40.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:40.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.457937 21881 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:40.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.463696 21881 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:40.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.464295 21881 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:40.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.568385 21881 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.253:34589
03:03:40.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.568497 21990 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.253:34589 every 8 connection(s)
03:03:40.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.571707 21881 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-1/data/info.pb
03:03:40.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.576994 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 21881
03:03:40.577 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:40.578 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.20.39.252:39247
03:03:40.578 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
03:03:40.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.580008 20639 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/logs
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/info.pb
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.252:39247
03:03:40.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.252
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34383
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.252
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
03:03:40.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.589432 21991 sys_catalog.cc:263] Verifying existing consensus state
03:03:40.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.600201 21991 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Bootstrap starting.
03:03:40.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.649477 21991 log.cc:826] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Log is configured to *not* fsync() on all Append() calls
03:03:40.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.668500 21991 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:03:40.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.669526 21991 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: Bootstrap complete.
03:03:40.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.704316 21991 raft_consensus.cc:359] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:40.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.707950 21991 raft_consensus.cc:740] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2bca0734326443059a4ce24d90982e7b, State: Initialized, Role: FOLLOWER
03:03:40.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.708957 21991 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:40.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.712577 21996 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:40.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.713658 21996 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: This master's current role is: FOLLOWER
03:03:40.714 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.714488 21991 sys_catalog.cc:565] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: configured and running, proceeding with master startup.
03:03:40.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.752224 22008 catalog_manager.cc:1269] Loaded cluster ID: fd29d4ee9aeb423086fd7ab91f0a3ae4
03:03:40.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.752538 22008 catalog_manager.cc:1562] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: loading cluster ID for follower catalog manager: success
03:03:40.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.763939 22008 catalog_manager.cc:1584] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: acquiring CA information for follower catalog manager: success
03:03:40.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.769924 22008 catalog_manager.cc:1612] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
03:03:40.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:40 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975820, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.253@KRBTEST.COM for kudu/127.20.39.254@KRBTEST.COM
03:03:40.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.835251 21881 master_runner.cc:186] Error getting master registration for 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111),
03:03:40.864 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.864009 21881 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:40.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.960947 21278 heartbeater.cc:344] Connected to a master server at 127.20.39.253:34589
03:03:40.968 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.967501 21954 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "631f0c8461bb42abad361a00af86e02d" instance_seqno: 1773975802555394) as {username='kudu', principal='kudu/127.20.39.194@KRBTEST.COM'} at 127.20.39.194:56027; Asking this server to re-register.
03:03:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.969013 21278 heartbeater.cc:461] Registering TS with master...
03:03:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:40.973122 21954 ts_manager.cc:194] Registered new tserver with Master: 631f0c8461bb42abad361a00af86e02d (127.20.39.194:39319)
03:03:41.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.124683 21993 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
03:03:41.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.125366 21993 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
03:03:41.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.126426 21993 flags.cc:425] Enabled unsafe flag: --never_fsync=true
03:03:41.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.175537 21993 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
03:03:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.175829 21993 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
03:03:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.176272 21993 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
03:03:41.176 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.176625 21993 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
03:03:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.176966 21993 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
03:03:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.177318 21993 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
03:03:41.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.177621 21993 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41191/jwks.json
03:03:41.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.220624 21993 master_runner.cc:386] Master server non-default flags:
03:03:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
03:03:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.20.39.212:34171
03:03:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
03:03:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
03:03:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data
03:03:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
03:03:41.222 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
03:03:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.20.39.254:34561,127.20.39.253:34589,127.20.39.252:39247
03:03:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/krb5kdc/kudu.keytab
03:03:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.20.39.252
03:03:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
03:03:41.223 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
03:03:41.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
03:03:41.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.20.39.252:39247
03:03:41.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
03:03:41.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
03:03:41.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
03:03:41.224 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41191/jwks.json
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/info.pb
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.20.39.252
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=34383
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/logs
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 3a5468b651ad64ac6e09f94bae9fffde09c4211d
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 20 Mar 2026 02:43:23 UTC on e7f111948823
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 11015
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.222381 21993 env_posix.cc:2267] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
03:03:41.225 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.223816 21993 file_cache.cc:492] Constructed file cache file cache with capacity 419430
03:03:41.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.254292 22019 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:41.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.325016 22025 raft_consensus.cc:493] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
03:03:41.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.326866 22025 raft_consensus.cc:515] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:41.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.357903 22025 leader_election.cc:290] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 2bca0734326443059a4ce24d90982e7b (127.20.39.253:34589), a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247)
03:03:41.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.365559 21964 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b521a42ad01a4b6aa3e36e623985f78b" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "2bca0734326443059a4ce24d90982e7b" is_pre_election: true
03:03:41.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.367538 21964 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate b521a42ad01a4b6aa3e36e623985f78b in term 2.
03:03:41.370 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.369805 21830 leader_election.cc:304] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2bca0734326443059a4ce24d90982e7b, b521a42ad01a4b6aa3e36e623985f78b; no voters:
03:03:41.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.377307 22025 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Leader pre-election won for term 3
03:03:41.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.378844 22025 raft_consensus.cc:493] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
03:03:41.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.380692 22025 raft_consensus.cc:3060] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 2 FOLLOWER]: Advancing to term 3
03:03:41.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.419117 21829 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.20.39.252:39247, user_credentials={real_user=slave}} blocked reactor thread for 58572.4us
03:03:41.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.443415 21126 heartbeater.cc:344] Connected to a master server at 127.20.39.253:34589
03:03:41.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.450464 21954 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "4169d8ccfba842878f17e1b90ec3855b" instance_seqno: 1773975799860149) as {username='kudu', principal='kudu/127.20.39.193@KRBTEST.COM'} at 127.20.39.193:41515; Asking this server to re-register.
03:03:41.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.452924 21126 heartbeater.cc:461] Registering TS with master...
03:03:41.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.453018 21435 heartbeater.cc:344] Connected to a master server at 127.20.39.253:34589
03:03:41.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.457314 21954 ts_manager.cc:194] Registered new tserver with Master: 4169d8ccfba842878f17e1b90ec3855b (127.20.39.193:35247)
03:03:41.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.460178 21953 master_service.cc:438] Got heartbeat from unknown tserver (permanent_uuid: "c7f0f6f6dcae46e3b8a714ebd1f6b86b" instance_seqno: 1773975805020399) as {username='kudu', principal='kudu/127.20.39.195@KRBTEST.COM'} at 127.20.39.195:33303; Asking this server to re-register.
03:03:41.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.461606 21435 heartbeater.cc:461] Registering TS with master...
03:03:41.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.465755 21953 ts_manager.cc:194] Registered new tserver with Master: c7f0f6f6dcae46e3b8a714ebd1f6b86b (127.20.39.195:34001)
03:03:41.472 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.443809 21830 outbound_call.cc:321] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.20.39.253:34589, user_credentials={real_user=slave}} blocked reactor thread for 74193.1us
03:03:41.484 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.484153 21829 leader_election.cc:336] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247): Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:41.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.517922 22025 raft_consensus.cc:515] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:41.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.538043 21964 tablet_service.cc:1917] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "b521a42ad01a4b6aa3e36e623985f78b" candidate_term: 3 candidate_status { last_received { term: 2 index: 5 } } ignore_live_leader: false dest_uuid: "2bca0734326443059a4ce24d90982e7b"
03:03:41.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.538995 21964 raft_consensus.cc:3060] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 2 FOLLOWER]: Advancing to term 3
03:03:41.584 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.584049 21829 leader_election.cc:336] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247): Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111)
03:03:41.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.595889 22025 leader_election.cc:290] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 3 election: Requested vote from peers 2bca0734326443059a4ce24d90982e7b (127.20.39.253:34589), a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247)
03:03:41.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.608719 21964 raft_consensus.cc:2468] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b521a42ad01a4b6aa3e36e623985f78b in term 3.
03:03:41.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.611196 21830 leader_election.cc:304] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2bca0734326443059a4ce24d90982e7b, b521a42ad01a4b6aa3e36e623985f78b; no voters: a5ecd815e9f94085b06bcd2fa458c820
03:03:41.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.612804 22025 raft_consensus.cc:2804] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 3 FOLLOWER]: Leader election won for term 3
03:03:41.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.613595 22025 raft_consensus.cc:697] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [term 3 LEADER]: Becoming Leader. State: Replica: b521a42ad01a4b6aa3e36e623985f78b, State: Running, Role: LEADER
03:03:41.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.614631 22025 consensus_queue.cc:237] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:41.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.636907 22029 sys_catalog.cc:455] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: SysCatalogTable state changed. Reason: New leader b521a42ad01a4b6aa3e36e623985f78b. Latest consensus state: current_term: 3 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:41.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.637609 22029 sys_catalog.cc:458] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: This master's current role is: LEADER
03:03:41.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.648660 22037 catalog_manager.cc:1485] Loading table and tablet metadata into memory...
03:03:41.685 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.685097 22037 catalog_manager.cc:1494] Initializing Kudu cluster ID...
03:03:41.708 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.707793 22037 catalog_manager.cc:1269] Loaded cluster ID: fd29d4ee9aeb423086fd7ab91f0a3ae4
03:03:41.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.715728 22037 catalog_manager.cc:1505] Initializing Kudu internal certificate authority...
03:03:41.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.756501 22037 catalog_manager.cc:1514] Loading token signing keys...
03:03:41.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.766393 22037 catalog_manager.cc:6055] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b: Loaded TSK: 0
03:03:41.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:41.768129 22037 catalog_manager.cc:1524] Initializing in-progress tserver states...
03:03:42.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:42.045154 21829 consensus_peers.cc:597] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b -> Peer a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247): Couldn't send request to peer a5ecd815e9f94085b06bcd2fa458c820. Status: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
03:03:42.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.096961 21964 raft_consensus.cc:1275] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [term 3 FOLLOWER]: Refusing update from remote peer b521a42ad01a4b6aa3e36e623985f78b: Log matching property violated. Preceding OpId in replica: term: 2 index: 5. Preceding OpId from leader: term: 3 index: 6. (index mismatch)
03:03:42.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.099820 22025 consensus_queue.cc:1048] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [LEADER]: Connected to new peer: Peer: permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
03:03:42.166 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.160562 22040 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: SysCatalogTable state changed. Reason: New leader b521a42ad01a4b6aa3e36e623985f78b. Latest consensus state: current_term: 3 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:42.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.168669 22029 sys_catalog.cc:455] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:42.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.169364 22029 sys_catalog.cc:458] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b [sys.catalog]: This master's current role is: LEADER
03:03:42.174 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.173830 22040 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: This master's current role is: FOLLOWER
03:03:42.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.178331 22040 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:42.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.179339 22040 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2bca0734326443059a4ce24d90982e7b [sys.catalog]: This master's current role is: FOLLOWER
03:03:42.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:42 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975822, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:42.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.330613 21158 init.cc:295] Successfully reacquired a new kerberos TGT
03:03:42.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.822335 21800 master_runner.cc:186] Error getting master registration for 127.20.39.252:39247: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111),
03:03:42.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.822978 21800 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:42.879 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:42.878841 21280 heartbeater.cc:499] Master 127.20.39.254:34561 was elected leader, sending a full tablet report...
03:03:43.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:43.113399 20624 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5a00 after lost signal to thread 20497
03:03:43.115 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:43.114976 20624 debug-util.cc:398] Leaking SignalData structure 0x7b08000b5e00 after lost signal to thread 20627
03:03:43.360 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:43.359257 21438 heartbeater.cc:499] Master 127.20.39.254:34561 was elected leader, sending a full tablet report...
03:03:43.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:43.540342 21133 heartbeater.cc:499] Master 127.20.39.254:34561 was elected leader, sending a full tablet report...
03:03:44.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:44.245180 21829 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111) [suppressed 7 similar messages]
03:03:44.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:44.505952 21993 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 3.248s user 0.001s sys 0.002s
03:03:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:44.506392 21993 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 3.249s user 0.001s sys 0.002s
03:03:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:42.652324 22018 debug-util.cc:398] Leaking SignalData structure 0x7b08000281c0 after lost signal to thread 21993
03:03:44.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:44.507298 22021 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 3247 milliseconds
03:03:44.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:41.256731 22020 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:44.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.517939 21993 server_base.cc:1056] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
03:03:44.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:44.518692 22023 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
03:03:44.520 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.520287 21993 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
03:03:44.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.528329 21993 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
03:03:44.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.529861 21993 hybrid_clock.cc:648] HybridClock initialized: now 1773975824529756 us; error 93 us; skew 500 ppm
03:03:44.531 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.531044 21993 server_base.cc:861] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
03:03:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:44 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975824, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:44.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.539206 21993 init.cc:377] Logged in from keytab as kudu/127.20.39.252@KRBTEST.COM (short username kudu)
03:03:44.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.557618 21993 webserver.cc:492] Webserver started at http://127.20.39.252:34383/ using document root <none> and password file <none>
03:03:44.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.558851 21993 fs_manager.cc:362] Metadata directory not provided
03:03:44.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.559134 21993 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
03:03:44.573 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.572985 21993 fs_manager.cc:714] Time spent opening directory manager: real 0.010s user 0.005s sys 0.003s
03:03:44.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.603701 22068 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
03:03:44.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.611563 21993 fs_manager.cc:730] Time spent opening block manager: real 0.036s user 0.005s sys 0.024s
03:03:44.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.611996 21993 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data,/tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:44.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a5ecd815e9f94085b06bcd2fa458c820"
03:03:44.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-03-20 03:03:14 on dist-test-slave-sx75"
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.614727 21993 fs_report.cc:389] FS layout report
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/wal
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/data
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
03:03:44.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
03:03:44.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:44.668965 21829 consensus_peers.cc:597] T 00000000000000000000000000000000 P b521a42ad01a4b6aa3e36e623985f78b -> Peer a5ecd815e9f94085b06bcd2fa458c820 (127.20.39.252:39247): Couldn't send request to peer a5ecd815e9f94085b06bcd2fa458c820. Status: Network error: Client connection negotiation failed: client connection to 127.20.39.252:39247: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
03:03:44.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:44 dist-test-slave-sx75 krb5kdc[20651](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975824, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
03:03:44.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.675846 21315 init.cc:295] Successfully reacquired a new kerberos TGT
03:03:44.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.694481 21993 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
03:03:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.707230 21993 env_posix.cc:2267] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
03:03:44.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.707780 21993 kserver.cc:163] Server-wide thread pool size limit: 3276
03:03:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.829720 21993 rpc_server.cc:307] RPC server started. Bound to: 127.20.39.252:39247
03:03:44.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.829864 22120 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.20.39.252:39247 every 8 connection(s)
03:03:44.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.833732 21993 server_base.cc:1193] Dumped server information to /tmp/dist-test-taskoRAqye/test-tmp/mini-kudu-cluster9208441313868778344/master-2/data/info.pb
03:03:44.835 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
03:03:44.836 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.20.39.254:34561(127.20.39.254:34561)] connecting to peer
03:03:44.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.834836 20639 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu as pid 21993
03:03:44.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.858455 22121 sys_catalog.cc:263] Verifying existing consensus state
03:03:44.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.866343 22121 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Bootstrap starting.
03:03:44.868 [DEBUG - kudu-nio-3] (Connection.java:757) [peer master-127.20.39.254:34561(127.20.39.254:34561)] Successfully connected to peer
03:03:44.891 [DEBUG - kudu-nio-3] (Connection.java:709) [peer master-127.20.39.254:34561(127.20.39.254:34561)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
03:03:44.896 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.39.254:34561(127.20.39.254:34561)] disconnecting while in state TERMINATED
03:03:44.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:44.894024 22124 negotiation.cc:337] Failed RPC negotiation. Trace:
03:03:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:44.860632 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:58174 (local address 127.20.39.254:34561)
03:03:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:44.871925 (+ 11293us) server_negotiation.cc:207] Beginning negotiation
03:03:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:44.871954 (+ 29us) server_negotiation.cc:400] Waiting for connection header
03:03:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:44.882670 (+ 10716us) server_negotiation.cc:408] Connection header received
03:03:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:44.888906 (+ 6236us) server_negotiation.cc:366] Received NEGOTIATE NegotiatePB request
03:03:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:44.888936 (+ 30us) server_negotiation.cc:462] Received NEGOTIATE request from client
03:03:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:44.889203 (+ 267us) server_negotiation.cc:378] Sending NEGOTIATE NegotiatePB response
03:03:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:44.889614 (+ 411us) server_negotiation.cc:227] Negotiated authn=SASL
03:03:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:44.892605 (+ 2991us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:58174: BlockingRecv error: recv got EOF from 127.0.0.1:58174 (error 108)
03:03:44.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":10620,"thread_start_us":10577,"threads_started":1}
03:03:44.897 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.39.253:34589(127.20.39.253:34589)] disconnecting while in state TERMINATED
03:03:44.900 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.20.39.252:39247(127.20.39.252:39247)] disconnecting while in state TERMINATED
03:03:44.901 [DEBUG - kudu-nio-3] (Connection.java:239) [peer master-127.20.39.254:34561(127.20.39.254:34561)] handling channelInactive
03:03:44.905 [DEBUG - Time-limited test] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:03:44.928 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:03:44.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.932225 22121 log.cc:826] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Log is configured to *not* fsync() on all Append() calls
03:03:44.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.968852 22121 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
03:03:44.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:44.969952 22121 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820: Bootstrap complete.
03:03:45.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.007418 22121 raft_consensus.cc:359] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:45.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.011266 22121 raft_consensus.cc:740] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: a5ecd815e9f94085b06bcd2fa458c820, State: Initialized, Role: FOLLOWER
03:03:45.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.013844 22121 consensus_queue.cc:260] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 2.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } }
03:03:45.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.020671 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 20991
03:03:45.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.023517 22132 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.024410 22132 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: This master's current role is: FOLLOWER
03:03:45.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.027606 22121 sys_catalog.cc:565] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: configured and running, proceeding with master startup.
03:03:45.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.100442 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 21142
03:03:45.179 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.178748 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 21292
03:03:45.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Mar 20 03:03:45 dist-test-slave-sx75 krb5kdc[20651](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1773975817, etypes {rep=17 tkt=17 ses=17}, kudu/127.20.39.254@KRBTEST.COM for kudu/127.20.39.252@KRBTEST.COM
03:03:45.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.232628 22096 raft_consensus.cc:3060] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [term 2 FOLLOWER]: Advancing to term 3
03:03:45.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.274880 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 21800
03:03:45.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.302175 22147 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:45.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.303282 22147 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: This master's current role is: FOLLOWER
03:03:45.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.308382 22132 sys_catalog.cc:455] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b521a42ad01a4b6aa3e36e623985f78b. Latest consensus state: current_term: 3 leader_uuid: "b521a42ad01a4b6aa3e36e623985f78b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "b521a42ad01a4b6aa3e36e623985f78b" member_type: VOTER last_known_addr { host: "127.20.39.254" port: 34561 } } peers { permanent_uuid: "2bca0734326443059a4ce24d90982e7b" member_type: VOTER last_known_addr { host: "127.20.39.253" port: 34589 } } peers { permanent_uuid: "a5ecd815e9f94085b06bcd2fa458c820" member_type: VOTER last_known_addr { host: "127.20.39.252" port: 39247 } } }
03:03:45.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.309089 22132 sys_catalog.cc:458] T 00000000000000000000000000000000 P a5ecd815e9f94085b06bcd2fa458c820 [sys.catalog]: This master's current role is: FOLLOWER
03:03:45.321 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.321415 22148 catalog_manager.cc:805] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
03:03:45.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:45.331597 22071 connection.cc:537] server connection from 127.0.0.1:47702 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
03:03:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.333093 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 21881
03:03:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:45.338502 22149 negotiation.cc:337] Failed RPC negotiation. Trace:
03:03:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:45.324914 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.20.39.254:34561 (local address 127.0.0.1:58186)
03:03:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:45.335490 (+ 10576us) negotiation.cc:107] Waiting for socket to connect
03:03:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0320 03:03:45.335797 (+ 307us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.20.39.254:34561: connect: Connection reset by peer (error 104)
03:03:45.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":9823,"threads_started":1}
03:03:45.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.348524 21993 master_runner.cc:186] Error getting master registration for 127.20.39.254:34561: Network error: Client connection negotiation failed: client connection to 127.20.39.254:34561: connect: Connection reset by peer (error 104),
03:03:45.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.379817 21993 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
03:03:45.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:45.381906 20639 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 21993
03:03:46.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:46.480285 20137 debug-util.cc:398] Leaking SignalData structure 0x7b0800099940 after lost signal to thread 20067
03:03:46.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260320 03:03:46.481459 20137 debug-util.cc:398] Leaking SignalData structure 0x7b08000b3f00 after lost signal to thread 20140
03:03:46.687 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:03:46Z chronyd exiting
03:03:47.738 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
03:03:47.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:47.740952 19905 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 20210
03:03:47.770 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:47.769637 19905 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 20354
03:03:47.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:47.800288 19905 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 20496
03:03:47.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:47.841236 19905 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 19917
03:03:47.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:47.881881 19905 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 19987
03:03:47.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260320 03:03:47.916567 19905 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskoRAqye/build/tsan/bin/kudu with pid 20062
03:03:49.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-03-20T03:03:49Z chronyd exiting
03:03:50.119 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more
org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.lang.Thread.run(Thread.java:840)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.Negotiator.chooseAndInitializeSaslMech(Negotiator.java:565)
at org.apache.kudu.client.Negotiator.handleNegotiateResponse(Negotiator.java:433)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:367)
at org.apache.kudu.client.Negotiator.channelRead0(Negotiator.java:94)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:361)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:325)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:171)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:796)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:732)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:658)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:998)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
... 1 more