14:19:41.998 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
14:19:42.010 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskciA_xo/test-tmp
14:19:42.013 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:19:42.016 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskciA_xo/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:19:42.243 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
14:19:42.422 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:19:42.423 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:19:42.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:42.426967 13439 mini_oidc.cc:150] Starting JWKS server
14:19:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:42.430339 13439 webserver.cc:480] Webserver started at http://127.0.0.1:43823/ using document root <none> and password file <none>
14:19:42.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:42.430735 13439 mini_oidc.cc:186] Starting OIDC Discovery server
14:19:42.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:42.432356 13439 webserver.cc:480] Webserver started at http://127.0.0.1:45735/ using document root <none> and password file <none>
14:19:42.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-01T14:19:42Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:19:42.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-01T14:19:42Z Disabled control of system clock
14:19:42.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:42.491659 13439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:42.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:42.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/wal
14:19:42.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/data
14:19:42.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:42.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:42.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/logs
14:19:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/data/info.pb
14:19:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:42.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.254:45259
14:19:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.254
14:19:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:42.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:42.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:42.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:42.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43823/jwks.json with env {}
14:19:42.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:42.789252 13451 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:42.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:42.789830 13451 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:42.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:42.790518 13451 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:42.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:42.822156 13451 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:42.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:42.822459 13451 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:42.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:42.822659 13451 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:42.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:42.822847 13451 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:42.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:42.823136 13451 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:42.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:42.823378 13451 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43823/jwks.json
14:19:42.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:42.859163 13451 master_runner.cc:386] Master server non-default flags:
14:19:42.859 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:42.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:42.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:42.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:42.860 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/data
14:19:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/wal
14:19:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.254:45259
14:19:42.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43823/jwks.json
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/data/info.pb
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.254
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:42.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/logs
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:42.860690 13451 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:42.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:42.862915 13451 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:42.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:42.875671 13457 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:44.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:42.877655 13458 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:44.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.090363 13460 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:44.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.093261 13459 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1211 milliseconds
14:19:44.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.093369 13451 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:19:44.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.094718 13451 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:44.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.097728 13451 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:44.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.099129 13451 hybrid_clock.cc:648] HybridClock initialized: now 1756736384099085 us; error 51 us; skew 500 ppm
14:19:44.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.100001 13451 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:44.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.106536 13451 webserver.cc:480] Webserver started at http://127.13.31.254:37491/ using document root <none> and password file <none>
14:19:44.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.107481 13451 fs_manager.cc:362] Metadata directory not provided
14:19:44.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.107673 13451 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:44.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.108055 13451 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:19:44.112 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.112465 13451 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/data/instance:
14:19:44.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "68819483133a4580bdaeb996abf56118"
14:19:44.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:44 on dist-test-slave-k2rc"
14:19:44.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.113683 13451 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/wal/instance:
14:19:44.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "68819483133a4580bdaeb996abf56118"
14:19:44.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:44 on dist-test-slave-k2rc"
14:19:44.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.120724 13451 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
14:19:44.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.126219 13467 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.127442 13451 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:19:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.127803 13451 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/wal
14:19:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "68819483133a4580bdaeb996abf56118"
14:19:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:44 on dist-test-slave-k2rc"
14:19:44.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.128201 13451 fs_report.cc:389] FS layout report
14:19:44.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:44.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/wal
14:19:44.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/wal
14:19:44.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/data/data
14:19:44.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:44.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:44.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:44.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:44.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.183404 13451 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:44.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.185060 13451 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:44.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.185580 13451 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:44.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.254792 13451 rpc_server.cc:307] RPC server started. Bound to: 127.13.31.254:45259
14:19:44.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.254889 13518 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.31.254:45259 every 8 connection(s)
14:19:44.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.257623 13451 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/data/info.pb
14:19:44.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.262732 13519 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:19:44.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.264236 13439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 13451
14:19:44.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.264688 13439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-0/wal/instance
14:19:44.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.268081 13439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:44.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:44.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/wal
14:19:44.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/data
14:19:44.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:44.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:44.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:44.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:44.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:44.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:44.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/logs
14:19:44.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/data/info.pb
14:19:44.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:44.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.253:37339
14:19:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.253
14:19:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:44.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:44.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:44.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:44.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:44.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:44.274 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43823/jwks.json with env {}
14:19:44.277 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.274214 13519 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } has no permanent_uuid. Determining permanent_uuid...
14:19:44.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.302814 13519 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } has no permanent_uuid. Determining permanent_uuid...
14:19:44.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.305148 13471 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.31.253:37339: connect: Connection refused (error 111)
14:19:44.309 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.309157 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.253:37339: Network error: Client connection negotiation failed: client connection to 127.13.31.253:37339: connect: Connection refused (error 111)
14:19:44.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.358654 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } attempt: 1
14:19:44.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.363441 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.253:37339: Network error: Client connection negotiation failed: client connection to 127.13.31.253:37339: connect: Connection refused (error 111)
14:19:44.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.431840 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } attempt: 2
14:19:44.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.436369 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.253:37339: Network error: Client connection negotiation failed: client connection to 127.13.31.253:37339: connect: Connection refused (error 111)
14:19:44.528 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.527796 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } attempt: 3
14:19:44.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.531905 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.253:37339: Network error: Client connection negotiation failed: client connection to 127.13.31.253:37339: connect: Connection refused (error 111)
14:19:44.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.577257 13521 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:44.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.578016 13521 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:44.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.578790 13521 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:44.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.610208 13521 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:44.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.610572 13521 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:44.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.610906 13521 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:44.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.611253 13521 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:44.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.611645 13521 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:44.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.611881 13521 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43823/jwks.json
14:19:44.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.650976 13521 master_runner.cc:386] Master server non-default flags:
14:19:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/data
14:19:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/wal
14:19:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:44.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.253:37339
14:19:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43823/jwks.json
14:19:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:44.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/data/info.pb
14:19:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.253
14:19:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:44.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/logs
14:19:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:19:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:19:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:19:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:19:44.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:19:44.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.652747 13521 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:44.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.655025 13521 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:44.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.666289 13530 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:44.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.675277 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } attempt: 4
14:19:44.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.678840 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.253:37339: Network error: Client connection negotiation failed: client connection to 127.13.31.253:37339: connect: Connection refused (error 111)
14:19:44.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:44.978255 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } attempt: 5
14:19:44.985 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.984586 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.253:37339: Network error: Client connection negotiation failed: client connection to 127.13.31.253:37339: connect: Connection refused (error 111)
14:19:45.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.532204 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } attempt: 6
14:19:45.545 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:45.544467 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.253:37339: Network error: Client connection negotiation failed: client connection to 127.13.31.253:37339: connect: Connection refused (error 111)
14:19:45.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.670430 13533 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:45.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:44.667552 13531 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:45.889 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:45.889524 13532 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1217 milliseconds
14:19:45.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.889627 13521 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:19:45.891 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.891147 13521 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:45.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.894461 13521 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:45.896 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.895852 13521 hybrid_clock.cc:648] HybridClock initialized: now 1756736385895804 us; error 51 us; skew 500 ppm
14:19:45.897 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.896962 13521 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:45.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.903663 13521 webserver.cc:480] Webserver started at http://127.13.31.253:39467/ using document root <none> and password file <none>
14:19:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.904518 13521 fs_manager.cc:362] Metadata directory not provided
14:19:45.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.904716 13521 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:45.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.905140 13521 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:19:45.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.909442 13521 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/data/instance:
14:19:45.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "769247e3dd6a4580af91fb62024e8ad8"
14:19:45.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:45 on dist-test-slave-k2rc"
14:19:45.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.910950 13521 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/wal/instance:
14:19:45.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "769247e3dd6a4580af91fb62024e8ad8"
14:19:45.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:45 on dist-test-slave-k2rc"
14:19:45.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.917855 13521 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.001s
14:19:45.923 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.922987 13542 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.924039 13521 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
14:19:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.924371 13521 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/wal
14:19:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "769247e3dd6a4580af91fb62024e8ad8"
14:19:45.924 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:45 on dist-test-slave-k2rc"
14:19:45.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.924727 13521 fs_report.cc:389] FS layout report
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/wal
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/wal
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/data/data
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:45.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:45.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:45.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.971901 13521 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:45.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.973448 13521 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:45.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:45.973938 13521 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:46.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.043820 13521 rpc_server.cc:307] RPC server started. Bound to: 127.13.31.253:37339
14:19:46.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.043905 13593 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.31.253:37339 every 8 connection(s)
14:19:46.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.046571 13521 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/data/info.pb
14:19:46.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.051486 13594 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:19:46.052 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.052371 13439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 13521
14:19:46.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.052805 13439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-1/wal/instance
14:19:46.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.054845 13439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:46.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:46.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/wal
14:19:46.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/data
14:19:46.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:46.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:46.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:46.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:46.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:46.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:46.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/logs
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/data/info.pb
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:46.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.252:44535
14:19:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.252
14:19:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:46.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43823/jwks.json with env {}
14:19:46.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.064479 13594 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } has no permanent_uuid. Determining permanent_uuid...
14:19:46.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.086989 13594 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } has no permanent_uuid. Determining permanent_uuid...
14:19:46.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.101383 13594 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } has no permanent_uuid. Determining permanent_uuid...
14:19:46.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.103930 13545 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:46.108 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.108233 13594 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:46.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.157697 13594 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 1
14:19:46.161 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.161502 13594 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:46.230 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.229837 13594 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 2
14:19:46.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.233872 13594 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:46.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.325242 13594 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 3
14:19:46.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.330067 13594 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:46.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.361111 13596 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:46.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.361649 13596 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:46.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.362296 13596 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:46.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.392979 13596 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:46.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.393247 13596 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:46.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.393445 13596 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:46.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.393640 13596 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:46.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.393831 13596 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:46.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.393994 13596 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:43823/jwks.json
14:19:46.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.429226 13596 master_runner.cc:386] Master server non-default flags:
14:19:46.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/data
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/wal
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.252:44535
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:43823/jwks.json
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/data/info.pb
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.252
14:19:46.430 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/logs
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:19:46.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:19:46.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:46.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:19:46.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:19:46.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:19:46.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.430730 13596 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:46.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.432631 13596 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:46.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.443198 13606 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:46.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.473469 13594 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 4
14:19:46.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.480186 13594 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:46.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.605037 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } attempt: 7
14:19:46.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.637845 13519 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } has no permanent_uuid. Determining permanent_uuid...
14:19:46.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.646389 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:46.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.705039 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 1
14:19:46.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.711357 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:46.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.779783 13594 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 5
14:19:46.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.786202 13594 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:46.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.792950 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 2
14:19:46.801 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.801270 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:46.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:46.886852 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 3
14:19:46.894 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.893582 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:47.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.034148 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 4
14:19:47.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:47.040472 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:47.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.324050 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 5
14:19:47.330 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:47.330380 13519 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:47.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.333752 13594 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 6
14:19:47.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:47.344437 13594 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.31.252:44535: Network error: Client connection negotiation failed: client connection to 127.13.31.252:44535: connect: Connection refused (error 111)
14:19:47.665 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:46.445331 13607 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:47.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:47.667358 13609 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:47.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:47.669641 13608 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1220 milliseconds
14:19:47.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.669786 13596 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:19:47.671 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.671066 13596 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:47.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.673594 13596 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:47.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.674935 13596 hybrid_clock.cc:648] HybridClock initialized: now 1756736387674907 us; error 35 us; skew 500 ppm
14:19:47.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.675747 13596 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:47.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.682889 13596 webserver.cc:480] Webserver started at http://127.13.31.252:38151/ using document root <none> and password file <none>
14:19:47.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.683748 13596 fs_manager.cc:362] Metadata directory not provided
14:19:47.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.683938 13596 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:47.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.684388 13596 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:19:47.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.689107 13596 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/data/instance:
14:19:47.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6717a918c0844e6922f6296382b2a9d"
14:19:47.689 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:47 on dist-test-slave-k2rc"
14:19:47.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.690295 13596 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/wal/instance:
14:19:47.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6717a918c0844e6922f6296382b2a9d"
14:19:47.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:47 on dist-test-slave-k2rc"
14:19:47.697 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.697324 13596 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.001s
14:19:47.703 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.703249 13621 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:47.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.704211 13596 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
14:19:47.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.704571 13596 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/wal
14:19:47.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "a6717a918c0844e6922f6296382b2a9d"
14:19:47.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:47 on dist-test-slave-k2rc"
14:19:47.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.704924 13596 fs_report.cc:389] FS layout report
14:19:47.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:47.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/wal
14:19:47.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/wal
14:19:47.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/data/data
14:19:47.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:47.705 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:47.706 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:47.771 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.771166 13596 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:47.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.772737 13596 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:47.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.773231 13596 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:47.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.843987 13596 rpc_server.cc:307] RPC server started. Bound to: 127.13.31.252:44535
14:19:47.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.844060 13672 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.31.252:44535 every 8 connection(s)
14:19:47.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.846925 13596 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/data/info.pb
14:19:47.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.852082 13673 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:19:47.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.854128 13439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 13596
14:19:47.854 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.854452 13439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/master-2/wal/instance
14:19:47.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.866510 13673 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } has no permanent_uuid. Determining permanent_uuid...
14:19:47.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.883111 13519 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 6
14:19:47.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.901042 13673 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } has no permanent_uuid. Determining permanent_uuid...
14:19:47.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.914484 13673 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } has no permanent_uuid. Determining permanent_uuid...
14:19:47.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.920452 13519 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118: Bootstrap starting.
14:19:47.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.927768 13519 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118: Neither blocks nor log segments found. Creating new log.
14:19:47.930 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.929911 13519 log.cc:826] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118: Log is configured to *not* fsync() on all Append() calls
14:19:47.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.934262 13519 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118: No bootstrap required, opened a new log
14:19:47.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.934248 13673 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d: Bootstrap starting.
14:19:47.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.941926 13673 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d: Neither blocks nor log segments found. Creating new log.
14:19:47.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.944078 13673 log.cc:826] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d: Log is configured to *not* fsync() on all Append() calls
14:19:47.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.950759 13673 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d: No bootstrap required, opened a new log
14:19:47.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.955772 13519 raft_consensus.cc:357] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } }
14:19:47.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.957063 13519 raft_consensus.cc:383] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:19:47.957 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.957545 13519 raft_consensus.cc:738] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 68819483133a4580bdaeb996abf56118, State: Initialized, Role: FOLLOWER
14:19:47.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.958730 13519 consensus_queue.cc:260] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } }
14:19:47.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.961825 13685 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } } }
14:19:47.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.962836 13685 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [sys.catalog]: This master's current role is: FOLLOWER
14:19:47.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.963704 13519 sys_catalog.cc:564] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [sys.catalog]: configured and running, proceeding with master startup.
14:19:47.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.972992 13673 raft_consensus.cc:357] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } }
14:19:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.973874 13673 raft_consensus.cc:383] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:19:47.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.974179 13673 raft_consensus.cc:738] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a6717a918c0844e6922f6296382b2a9d, State: Initialized, Role: FOLLOWER
14:19:47.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.974920 13673 consensus_queue.cc:260] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } }
14:19:47.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.977927 13692 sys_catalog.cc:455] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } } }
14:19:47.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.979789 13692 sys_catalog.cc:458] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [sys.catalog]: This master's current role is: FOLLOWER
14:19:47.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.979866 13673 sys_catalog.cc:564] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [sys.catalog]: configured and running, proceeding with master startup.
14:19:48.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:48.004324 13700 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:48.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:48.005728 13700 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:48.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:47.995663 13451 master_runner.cc:186] Error getting master registration for 127.13.31.253:37339: OK, instance_id { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" instance_seqno: 1756736386024776 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:19:48.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.026619 13451 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:19:48.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:48.028362 13627 tablet.cc:2378] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:19:48.031 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:48.031333 13709 catalog_manager.cc:1560] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:19:48.032 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:48.031838 13709 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:19:48.056 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.023659 13596 master_runner.cc:186] Error getting master registration for 127.13.31.253:37339: OK, instance_id { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" instance_seqno: 1756736386024776 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:19:48.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.066145 13596 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:19:48.074 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.067225 13692 raft_consensus.cc:491] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:19:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.067762 13692 raft_consensus.cc:513] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } }
14:19:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.072700 13494 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a6717a918c0844e6922f6296382b2a9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "68819483133a4580bdaeb996abf56118" is_pre_election: true
14:19:48.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.073642 13494 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a6717a918c0844e6922f6296382b2a9d in term 0.
14:19:48.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.076078 13625 leader_election.cc:304] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 68819483133a4580bdaeb996abf56118, a6717a918c0844e6922f6296382b2a9d; no voters:
14:19:48.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.070276 13692 leader_election.cc:290] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 68819483133a4580bdaeb996abf56118 (127.13.31.254:45259), 769247e3dd6a4580af91fb62024e8ad8 (127.13.31.253:37339)
14:19:48.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.078685 13692 raft_consensus.cc:2802] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [term 0 FOLLOWER]: Leader pre-election won for term 1
14:19:48.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.079474 13692 raft_consensus.cc:491] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:19:48.080 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.080256 13692 raft_consensus.cc:3058] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [term 0 FOLLOWER]: Advancing to term 1
14:19:48.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.113785 13692 raft_consensus.cc:513] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } }
14:19:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.116570 13494 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a6717a918c0844e6922f6296382b2a9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "68819483133a4580bdaeb996abf56118"
14:19:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.117175 13494 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [term 0 FOLLOWER]: Advancing to term 1
14:19:48.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.124120 13494 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a6717a918c0844e6922f6296382b2a9d in term 1.
14:19:48.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.126816 13625 leader_election.cc:304] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 68819483133a4580bdaeb996abf56118, a6717a918c0844e6922f6296382b2a9d; no voters:
14:19:48.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.128062 13711 raft_consensus.cc:2802] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [term 1 FOLLOWER]: Leader election won for term 1
14:19:48.133 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.133386 13711 raft_consensus.cc:695] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [term 1 LEADER]: Becoming Leader. State: Replica: a6717a918c0844e6922f6296382b2a9d, State: Running, Role: LEADER
14:19:48.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.133555 13692 leader_election.cc:290] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [CANDIDATE]: Term 1 election: Requested vote from peers 68819483133a4580bdaeb996abf56118 (127.13.31.254:45259), 769247e3dd6a4580af91fb62024e8ad8 (127.13.31.253:37339)
14:19:48.135 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.134757 13711 consensus_queue.cc:237] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } }
14:19:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.143204 13712 sys_catalog.cc:455] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [sys.catalog]: SysCatalogTable state changed. Reason: New leader a6717a918c0844e6922f6296382b2a9d. Latest consensus state: current_term: 1 leader_uuid: "a6717a918c0844e6922f6296382b2a9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } } }
14:19:48.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.144021 13712 sys_catalog.cc:458] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [sys.catalog]: This master's current role is: LEADER
14:19:48.154 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.153995 13714 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:19:48.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.162828 13714 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:19:48.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.186475 13494 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [term 1 FOLLOWER]: Refusing update from remote peer a6717a918c0844e6922f6296382b2a9d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:19:48.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.188338 13711 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:19:48.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:48.194792 13473 tablet.cc:2378] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:19:48.228 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.227857 13685 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a6717a918c0844e6922f6296382b2a9d. Latest consensus state: current_term: 1 leader_uuid: "a6717a918c0844e6922f6296382b2a9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } } }
14:19:48.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.228642 13685 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [sys.catalog]: This master's current role is: FOLLOWER
14:19:48.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.233502 13715 mvcc.cc:204] Tried to move back new op lower bound from 7195592245989748736 to 7195592245810245632. Current Snapshot: MvccSnapshot[applied={T|T < 7195592245989748736}]
14:19:48.236 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.235906 13692 sys_catalog.cc:455] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a6717a918c0844e6922f6296382b2a9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } } }
14:19:48.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.236907 13692 sys_catalog.cc:458] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [sys.catalog]: This master's current role is: LEADER
14:19:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.238755 13714 catalog_manager.cc:1349] Generated new cluster ID: 376b2e9a99464b058bf0c65a70a9fe53
14:19:48.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.239048 13714 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:19:48.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.240335 13716 mvcc.cc:204] Tried to move back new op lower bound from 7195592245989748736 to 7195592245810245632. Current Snapshot: MvccSnapshot[applied={T|T < 7195592245989748736}]
14:19:48.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.250784 13685 sys_catalog.cc:455] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a6717a918c0844e6922f6296382b2a9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } } }
14:19:48.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.251443 13685 sys_catalog.cc:458] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118 [sys.catalog]: This master's current role is: FOLLOWER
14:19:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.289310 13714 catalog_manager.cc:1372] Generated new certificate authority record
14:19:48.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.291343 13714 catalog_manager.cc:1506] Loading token signing keys...
14:19:48.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.341893 13714 catalog_manager.cc:5955] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d: Generated new TSK 0
14:19:48.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.343796 13714 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:19:48.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.380434 13439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/wal
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/data
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/logs
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/data/info.pb
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:48.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:19:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.193:0
14:19:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.31.193
14:19:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.193
14:19:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:48.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:19:48.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.404963 13594 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } attempt: 7
14:19:48.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.441606 13594 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8: Bootstrap starting.
14:19:48.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.450477 13594 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8: Neither blocks nor log segments found. Creating new log.
14:19:48.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.453075 13594 log.cc:826] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8: Log is configured to *not* fsync() on all Append() calls
14:19:48.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.467284 13594 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8: No bootstrap required, opened a new log
14:19:48.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.495595 13594 raft_consensus.cc:357] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } }
14:19:48.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.496785 13594 raft_consensus.cc:383] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:19:48.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.497308 13594 raft_consensus.cc:738] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 769247e3dd6a4580af91fb62024e8ad8, State: Initialized, Role: FOLLOWER
14:19:48.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.498350 13594 consensus_queue.cc:260] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } }
14:19:48.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.511590 13568 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a6717a918c0844e6922f6296382b2a9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "769247e3dd6a4580af91fb62024e8ad8"
14:19:48.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.512367 13568 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [term 0 FOLLOWER]: Advancing to term 1
14:19:48.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.515659 13569 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "a6717a918c0844e6922f6296382b2a9d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "769247e3dd6a4580af91fb62024e8ad8" is_pre_election: true
14:19:48.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.521924 13568 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a6717a918c0844e6922f6296382b2a9d in term 1.
14:19:48.527 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.523399 13567 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [term 1 FOLLOWER]: Refusing update from remote peer a6717a918c0844e6922f6296382b2a9d: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:19:48.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.530192 13594 sys_catalog.cc:564] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [sys.catalog]: configured and running, proceeding with master startup.
14:19:48.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.531507 13692 consensus_queue.cc:1035] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [LEADER]: Connected to new peer: Peer: permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:19:48.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.504292 13723 sys_catalog.cc:455] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } } }
14:19:48.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.534072 13723 sys_catalog.cc:458] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [sys.catalog]: This master's current role is: FOLLOWER
14:19:48.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.606686 13723 sys_catalog.cc:455] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader a6717a918c0844e6922f6296382b2a9d. Latest consensus state: current_term: 1 leader_uuid: "a6717a918c0844e6922f6296382b2a9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } } }
14:19:48.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.607424 13723 sys_catalog.cc:458] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [sys.catalog]: This master's current role is: FOLLOWER
14:19:48.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.609362 13731 sys_catalog.cc:455] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "a6717a918c0844e6922f6296382b2a9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } } }
14:19:48.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.611828 13731 sys_catalog.cc:458] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8 [sys.catalog]: This master's current role is: FOLLOWER
14:19:48.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:48.628589 13740 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:19:48.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:48.979708 13720 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:48.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:48.980252 13720 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:48.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:48.980950 13720 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:49.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.011700 13700 catalog_manager.cc:1261] Loaded cluster ID: 376b2e9a99464b058bf0c65a70a9fe53
14:19:49.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.012301 13700 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118: loading cluster ID for follower catalog manager: success
14:19:49.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.020056 13700 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118: acquiring CA information for follower catalog manager: success
14:19:49.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:49.027439 13720 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:49.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:49.029338 13720 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.31.193
14:19:49.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.032691 13700 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 68819483133a4580bdaeb996abf56118: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:19:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.049137 13692 sys_catalog.cc:455] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "a6717a918c0844e6922f6296382b2a9d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "68819483133a4580bdaeb996abf56118" member_type: VOTER last_known_addr { host: "127.13.31.254" port: 45259 } } peers { permanent_uuid: "769247e3dd6a4580af91fb62024e8ad8" member_type: VOTER last_known_addr { host: "127.13.31.253" port: 37339 } } peers { permanent_uuid: "a6717a918c0844e6922f6296382b2a9d" member_type: VOTER last_known_addr { host: "127.13.31.252" port: 44535 } } }
14:19:49.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.049865 13692 sys_catalog.cc:458] T 00000000000000000000000000000000 P a6717a918c0844e6922f6296382b2a9d [sys.catalog]: This master's current role is: LEADER
14:19:49.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.100392 13720 tablet_server_runner.cc:78] Tablet server non-default flags:
14:19:49.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:49.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:49.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:49.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:49.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/data
14:19:49.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/wal
14:19:49.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.193:0
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/data/info.pb
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.193
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.31.193
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/logs
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:19:49.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:19:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:19:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:19:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:19:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:19:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.102200 13720 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:49.105 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.104451 13720 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:49.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:49.128201 13748 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:49.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.652269 13740 catalog_manager.cc:1261] Loaded cluster ID: 376b2e9a99464b058bf0c65a70a9fe53
14:19:49.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.653100 13740 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8: loading cluster ID for follower catalog manager: success
14:19:49.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.683527 13740 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8: acquiring CA information for follower catalog manager: success
14:19:49.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:49.689657 13740 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 769247e3dd6a4580af91fb62024e8ad8: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:19:50.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:49.131578 13749 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:50.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:50.530627 13747 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 13720
14:19:50.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:50.977478 13720 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 1.848s user 0.561s sys 1.092s
14:19:50.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:50.979418 13720 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 1.850s user 0.561s sys 1.092s
14:19:50.996 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:50.996316 13720 server_base.cc:1047] running on GCE node
14:19:50.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:50.997758 13754 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:50.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:50.999182 13720 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:51.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.006505 13720 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:51.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.007933 13720 hybrid_clock.cc:648] HybridClock initialized: now 1756736391007856 us; error 72 us; skew 500 ppm
14:19:51.009 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.009025 13720 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:51.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.037539 13720 webserver.cc:480] Webserver started at http://127.13.31.193:41457/ using document root <none> and password file <none>
14:19:51.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.038955 13720 fs_manager.cc:362] Metadata directory not provided
14:19:51.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.039306 13720 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:51.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.039961 13720 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:19:51.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.046761 13720 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/data/instance:
14:19:51.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ddfe23f388de45c7837feb1ea7af8fc0"
14:19:51.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:51 on dist-test-slave-k2rc"
14:19:51.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.048350 13720 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/wal/instance:
14:19:51.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ddfe23f388de45c7837feb1ea7af8fc0"
14:19:51.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:51 on dist-test-slave-k2rc"
14:19:51.070 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.070070 13720 fs_manager.cc:696] Time spent creating directory manager: real 0.021s user 0.014s sys 0.006s
14:19:51.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.087639 13759 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:51.090 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.089985 13720 fs_manager.cc:730] Time spent opening block manager: real 0.015s user 0.005s sys 0.009s
14:19:51.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.090667 13720 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/wal
14:19:51.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ddfe23f388de45c7837feb1ea7af8fc0"
14:19:51.091 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:51 on dist-test-slave-k2rc"
14:19:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.101660 13720 fs_report.cc:389] FS layout report
14:19:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/wal
14:19:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/wal
14:19:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/data/data
14:19:51.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:51.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:51.202 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.201989 13720 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.204022 13720 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:51.204 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.204591 13720 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:51.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.208817 13720 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:19:51.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.219362 13720 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:19:51.219 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.219746 13720 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:19:51.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.220057 13720 ts_tablet_manager.cc:610] Registered 0 tablets
14:19:51.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.220326 13720 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:19:51.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.521170 13720 rpc_server.cc:307] RPC server started. Bound to: 127.13.31.193:45439
14:19:51.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.521293 13871 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.31.193:45439 every 8 connection(s)
14:19:51.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.524747 13720 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/data/info.pb
14:19:51.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.531870 13439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 13720
14:19:51.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.532490 13439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-0/wal/instance
14:19:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.559182 13439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:51.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/wal
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/data
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/logs
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/data/info.pb
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.194:0
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.31.194
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.194
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:51.560 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:51.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:51.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:51.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:51.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:19:51.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.615878 13872 heartbeater.cc:344] Connected to a master server at 127.13.31.252:44535
14:19:51.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.616392 13872 heartbeater.cc:461] Registering TS with master...
14:19:51.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.617728 13872 heartbeater.cc:507] Master 127.13.31.252:44535 requested a full tablet report, sending...
14:19:51.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.621416 13638 ts_manager.cc:194] Registered new tserver with Master: ddfe23f388de45c7837feb1ea7af8fc0 (127.13.31.193:45439)
14:19:51.625 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.624053 13638 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.13.31.193:37525
14:19:51.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.638551 13873 heartbeater.cc:344] Connected to a master server at 127.13.31.253:37339
14:19:51.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.638957 13873 heartbeater.cc:461] Registering TS with master...
14:19:51.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.639649 13873 heartbeater.cc:507] Master 127.13.31.253:37339 requested a full tablet report, sending...
14:19:51.643 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.643003 13558 ts_manager.cc:194] Registered new tserver with Master: ddfe23f388de45c7837feb1ea7af8fc0 (127.13.31.193:45439)
14:19:51.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.650739 13877 heartbeater.cc:344] Connected to a master server at 127.13.31.254:45259
14:19:51.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.651108 13877 heartbeater.cc:461] Registering TS with master...
14:19:51.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.651753 13877 heartbeater.cc:507] Master 127.13.31.254:45259 requested a full tablet report, sending...
14:19:51.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:51.655625 13483 ts_manager.cc:194] Registered new tserver with Master: ddfe23f388de45c7837feb1ea7af8fc0 (127.13.31.193:45439)
14:19:51.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:51.965216 13876 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:51.965798 13876 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:51.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:51.966373 13876 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:51.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:51.999202 13876 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:52.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:52.000166 13876 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.31.194
14:19:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:52.034832 13876 tablet_server_runner.cc:78] Tablet server non-default flags:
14:19:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/data
14:19:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/wal
14:19:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.194:0
14:19:52.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/data/info.pb
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.194
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.31.194
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/logs
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:52.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:19:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:19:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:19:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:19:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:19:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:19:52.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:52.036515 13876 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:52.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:52.039448 13876 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:52.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:52.054795 13888 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:52.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:52.628641 13872 heartbeater.cc:499] Master 127.13.31.252:44535 was elected leader, sending a full tablet report...
14:19:53.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:53.458285 13887 debug-util.cc:398] Leaking SignalData structure 0x7b0800034ea0 after lost signal to thread 13876
14:19:53.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:53.628899 13876 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.571s user 0.485s sys 0.996s
14:19:53.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:53.629320 13876 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.572s user 0.485s sys 0.996s
14:19:53.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:52.057834 13889 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:53.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:53.630465 13890 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 1572 milliseconds
14:19:53.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.631300 13876 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:19:53.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:53.631330 13891 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:53.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.636175 13876 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:53.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.638240 13876 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:53.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.639578 13876 hybrid_clock.cc:648] HybridClock initialized: now 1756736393639527 us; error 51 us; skew 500 ppm
14:19:53.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.640321 13876 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:53.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.646179 13876 webserver.cc:480] Webserver started at http://127.13.31.194:33357/ using document root <none> and password file <none>
14:19:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.647065 13876 fs_manager.cc:362] Metadata directory not provided
14:19:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.647254 13876 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:53.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.647640 13876 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:19:53.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.652745 13876 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/data/instance:
14:19:53.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b30a538a27642959e57f592c710d41a"
14:19:53.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:53 on dist-test-slave-k2rc"
14:19:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.653920 13876 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/wal/instance:
14:19:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b30a538a27642959e57f592c710d41a"
14:19:53.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:53 on dist-test-slave-k2rc"
14:19:53.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.661056 13876 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
14:19:53.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.667104 13903 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.668170 13876 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.006s sys 0.000s
14:19:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.668442 13876 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/wal
14:19:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "6b30a538a27642959e57f592c710d41a"
14:19:53.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:53 on dist-test-slave-k2rc"
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.668745 13876 fs_report.cc:389] FS layout report
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/wal
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/wal
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/data/data
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:53.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:53.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.733330 13876 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:53.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.734735 13876 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:53.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.735108 13876 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:53.737 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.737586 13876 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:19:53.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.741681 13876 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:19:53.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.741868 13876 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:19:53.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.742058 13876 ts_tablet_manager.cc:610] Registered 0 tablets
14:19:53.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.742193 13876 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:19:53.882 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.882473 13876 rpc_server.cc:307] RPC server started. Bound to: 127.13.31.194:42979
14:19:53.883 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.882625 14015 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.31.194:42979 every 8 connection(s)
14:19:53.885 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.885267 13876 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/data/info.pb
14:19:53.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.887904 13439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 13876
14:19:53.893 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.888615 13439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-1/wal/instance
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.910676 13439 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/wal
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/data
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/logs
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/data/info.pb
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:53.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.195:0
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.31.195
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.195
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:53.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:53.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:53.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:53.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:53.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin with env {}
14:19:53.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.926757 14016 heartbeater.cc:344] Connected to a master server at 127.13.31.252:44535
14:19:53.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.927289 14016 heartbeater.cc:461] Registering TS with master...
14:19:53.929 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.928655 14016 heartbeater.cc:507] Master 127.13.31.252:44535 requested a full tablet report, sending...
14:19:53.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.931327 13638 ts_manager.cc:194] Registered new tserver with Master: 6b30a538a27642959e57f592c710d41a (127.13.31.194:42979)
14:19:53.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.933048 13638 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.13.31.194:58603
14:19:53.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.933759 14018 heartbeater.cc:344] Connected to a master server at 127.13.31.254:45259
14:19:53.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.934113 14018 heartbeater.cc:461] Registering TS with master...
14:19:53.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.934520 14017 heartbeater.cc:344] Connected to a master server at 127.13.31.253:37339
14:19:53.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.934823 14017 heartbeater.cc:461] Registering TS with master...
14:19:53.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.935108 14018 heartbeater.cc:507] Master 127.13.31.254:45259 requested a full tablet report, sending...
14:19:53.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.935617 14017 heartbeater.cc:507] Master 127.13.31.253:37339 requested a full tablet report, sending...
14:19:53.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.938217 13558 ts_manager.cc:194] Registered new tserver with Master: 6b30a538a27642959e57f592c710d41a (127.13.31.194:42979)
14:19:53.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:53.938200 13483 ts_manager.cc:194] Registered new tserver with Master: 6b30a538a27642959e57f592c710d41a (127.13.31.194:42979)
14:19:54.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:54.220911 14026 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:54.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:54.221356 14026 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:54.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:54.221818 14026 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:54.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:54.253327 14026 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:54.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:54.254272 14026 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.31.195
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:54.290871 14026 tablet_server_runner.cc:78] Tablet server non-default flags:
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.31.212:38019
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/data
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/wal
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.31.195:0
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/data/info.pb
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.31.195
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:54.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.31.254:45259,127.13.31.253:37339,127.13.31.252:44535
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.31.195
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/logs
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:19:54.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:54.292243 14026 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:54.294 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:54.294049 14026 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:54.306 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:54.306257 14032 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:54.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:54.939133 14016 heartbeater.cc:499] Master 127.13.31.252:44535 was elected leader, sending a full tablet report...
14:19:55.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:54.306939 14033 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:55.635 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:55.635120 14035 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:19:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:55.639133 14026 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.333s user 0.375s sys 0.944s
14:19:55.639 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:55.639524 14026 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.333s user 0.375s sys 0.944s
14:19:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:55.640223 14034 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1332 milliseconds
14:19:55.640 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.640236 14026 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:19:55.642 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.641726 14026 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:19:55.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.644362 14026 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:19:55.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.645850 14026 hybrid_clock.cc:648] HybridClock initialized: now 1756736395645810 us; error 42 us; skew 500 ppm
14:19:55.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.646674 14026 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:19:55.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.653678 14026 webserver.cc:480] Webserver started at http://127.13.31.195:33793/ using document root <none> and password file <none>
14:19:55.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.654568 14026 fs_manager.cc:362] Metadata directory not provided
14:19:55.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.654795 14026 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:19:55.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.655325 14026 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:19:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.660218 14026 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/data/instance:
14:19:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4b124fb9bb2483f8ea0745ae25f0ff4"
14:19:55.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:55 on dist-test-slave-k2rc"
14:19:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.661337 14026 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/wal/instance:
14:19:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4b124fb9bb2483f8ea0745ae25f0ff4"
14:19:55.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:55 on dist-test-slave-k2rc"
14:19:55.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.668726 14026 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.005s sys 0.001s
14:19:55.675 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.674793 14043 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:19:55.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.676118 14026 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:19:55.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.676441 14026 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/wal
14:19:55.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "b4b124fb9bb2483f8ea0745ae25f0ff4"
14:19:55.676 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:19:55 on dist-test-slave-k2rc"
14:19:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.676774 14026 fs_report.cc:389] FS layout report
14:19:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:19:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/wal
14:19:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/wal
14:19:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/data/data
14:19:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:19:55.677 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:19:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:19:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:19:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:19:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:19:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:19:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:19:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:19:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:19:55.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:19:55.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.748997 14026 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:19:55.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.750478 14026 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:19:55.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.750962 14026 kserver.cc:163] Server-wide thread pool size limit: 3276
14:19:55.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.753528 14026 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:19:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.757871 14026 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:19:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.758078 14026 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:19:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.758378 14026 ts_tablet_manager.cc:610] Registered 0 tablets
14:19:55.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.758548 14026 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:19:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.902923 14026 rpc_server.cc:307] RPC server started. Bound to: 127.13.31.195:41201
14:19:55.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.902980 14155 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.31.195:41201 every 8 connection(s)
14:19:55.905 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.905524 14026 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/data/info.pb
14:19:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.912786 13439 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 14026
14:19:55.916 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.913403 13439 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster5914368117356341114/ts-2/wal/instance
14:19:55.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.953159 14156 heartbeater.cc:344] Connected to a master server at 127.13.31.252:44535
14:19:55.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.953673 14156 heartbeater.cc:461] Registering TS with master...
14:19:55.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.955341 14156 heartbeater.cc:507] Master 127.13.31.252:44535 requested a full tablet report, sending...
14:19:55.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.957748 13638 ts_manager.cc:194] Registered new tserver with Master: b4b124fb9bb2483f8ea0745ae25f0ff4 (127.13.31.195:41201)
14:19:55.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.958626 14158 heartbeater.cc:344] Connected to a master server at 127.13.31.254:45259
14:19:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.958953 14158 heartbeater.cc:461] Registering TS with master...
14:19:55.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.959535 13638 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.13.31.195:42301
14:19:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.959812 14158 heartbeater.cc:507] Master 127.13.31.254:45259 requested a full tablet report, sending...
14:19:55.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.960162 14157 heartbeater.cc:344] Connected to a master server at 127.13.31.253:37339
14:19:55.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.960510 14157 heartbeater.cc:461] Registering TS with master...
14:19:55.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.961344 14157 heartbeater.cc:507] Master 127.13.31.253:37339 requested a full tablet report, sending...
14:19:55.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.962073 13483 ts_manager.cc:194] Registered new tserver with Master: b4b124fb9bb2483f8ea0745ae25f0ff4 (127.13.31.195:41201)
14:19:55.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.963773 13558 ts_manager.cc:194] Registered new tserver with Master: b4b124fb9bb2483f8ea0745ae25f0ff4 (127.13.31.195:41201)
14:19:55.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:55.978719 13439 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:19:55.989 [DEBUG - main] (MiniKuduCluster.java:195) Response:
14:19:55.991 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
14:19:56.001 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.13.31.254"
port: 45259
}
bound_http_address {
host: "127.13.31.254"
port: 37491
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.13.31.253"
port: 37339
}
bound_http_address {
host: "127.13.31.253"
port: 39467
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.13.31.252"
port: 44535
}
bound_http_address {
host: "127.13.31.252"
port: 38151
}
}
}
14:19:56.004 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:19:56.007 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.13.31.193"
port: 45439
}
bound_http_address {
host: "127.13.31.193"
port: 41457
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.13.31.194"
port: 42979
}
bound_http_address {
host: "127.13.31.194"
port: 33357
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.13.31.195"
port: 41201
}
bound_http_address {
host: "127.13.31.195"
port: 33793
}
}
}
14:19:56.007 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
14:19:56.172 [DEBUG - main] (CompositeStrategy.java:59) Unable to load modern Subject methods, relying only on legacy methods
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_275]
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51) ~[main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52) [main/:?]
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41) [main/:?]
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43) [main/:?]
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155) [main/:?]
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431) [main/:?]
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299) [main/:?]
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109) [main/:?]
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50) [junit-4.13.2.jar:4.13.2]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244) [kudu-test-utils-1.19.0-SNAPSHOT-unshaded.jar:1.19.0-SNAPSHOT]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:128) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.Suite.runChild(Suite.java:27) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) [junit-4.13.2.jar:4.13.2]
at org.junit.runners.ParentRunner.run(ParentRunner.java:413) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:137) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.run(JUnitCore.java:115) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77) [junit-4.13.2.jar:4.13.2]
at org.junit.runner.JUnitCore.main(JUnitCore.java:36) [junit-4.13.2.jar:4.13.2]
java.lang.NoSuchMethodException: javax.security.auth.Subject.current()
at java.lang.Class.getDeclaredMethod(Class.java:2130)
at org.apache.kudu.client.internals.ModernStrategy.<init>(ModernStrategy.java:51)
at org.apache.kudu.client.internals.CompositeStrategy.<init>(CompositeStrategy.java:52)
at org.apache.kudu.client.internals.CompositeStrategy.<clinit>(CompositeStrategy.java:41)
at org.apache.kudu.client.internals.SecurityManagerCompatibility.get(SecurityManagerCompatibility.java:43)
at org.apache.kudu.client.SecurityContext.setupSubject(SecurityContext.java:155)
at org.apache.kudu.client.SecurityContext.<init>(SecurityContext.java:138)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:431)
at org.apache.kudu.client.AsyncKuduClient.<init>(AsyncKuduClient.java:299)
at org.apache.kudu.client.AsyncKuduClient$AsyncKuduClientBuilder.build(AsyncKuduClient.java:3109)
at org.apache.kudu.test.KuduTestHarness.before(KuduTestHarness.java:161)
at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:50)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.doOneAttemptAndReport(RetryRule.java:184)
at org.apache.kudu.test.junit.RetryRule$RetryStatement.evaluate(RetryRule.java:244)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runners.Suite.runChild(Suite.java:128)
at org.junit.runners.Suite.runChild(Suite.java:27)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at org.junit.runner.JUnitCore.run(JUnitCore.java:115)
at org.junit.runner.JUnitCore.runMain(JUnitCore.java:77)
at org.junit.runner.JUnitCore.main(JUnitCore.java:36)
14:19:56.197 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
14:19:56.321 [INFO - Time-limited test] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskciA_xo/test-tmp
14:19:56.322 [INFO - Time-limited test] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/tsan/bin
14:19:56.322 [INFO - Time-limited test] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskciA_xo/java/build/dist-test/../../../build/tsan/bin/kudu, test, mini_cluster, --serialization=pb]
14:19:56.324 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: true
cluster_root: "/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847"
extra_master_flags: "--webserver_require_spnego=false"
extra_master_flags: "--authn_token_validity_seconds=10"
extra_tserver_flags: "--webserver_require_spnego=false"
hms_mode: NONE
mini_kdc_options {
ticket_lifetime: "10s"
renew_lifetime: "20s"
}
principal: "kudu"
mini_oidc_options {
}
}
14:19:56.654 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:19:56.654 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_cluster {
}
14:19:56.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:56.658164 14167 mini_oidc.cc:150] Starting JWKS server
14:19:56.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:56.661792 14167 webserver.cc:480] Webserver started at http://127.0.0.1:41679/ using document root <none> and password file <none>
14:19:56.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:56.662204 14167 mini_oidc.cc:186] Starting OIDC Discovery server
14:19:56.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:56.664434 14167 webserver.cc:480] Webserver started at http://127.0.0.1:42461/ using document root <none> and password file <none>
14:19:56.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Loading random data
14:19:56.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Initializing database '/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/principal' for realm 'KRBTEST.COM',
14:19:56.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master key name 'K/M@KRBTEST.COM'
14:19:56.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:19:56 dist-test-slave-k2rc krb5kdc[14180](info): setting up network...
14:19:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: setsockopt(10,IPV6_V6ONLY,1) worked
14:19:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:19:56 dist-test-slave-k2rc krb5kdc[14180](info): set up 2 sockets
14:19:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:19:56 dist-test-slave-k2rc krb5kdc[14180](info): commencing operation
14:19:56.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) krb5kdc: starting...
14:19:56.964 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:56.963665 14156 heartbeater.cc:499] Master 127.13.31.252:44535 was elected leader, sending a full tablet report...
14:19:58.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:58.768316 14167 mini_kdc.cc:121] Time spent starting KDC: real 2.066s user 0.007s sys 0.011s
14:19:58.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-admin@KRBTEST.COM; defaulting to no policy
14:19:58.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:19:58.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-admin@KRBTEST.COM" created.
14:19:58.810 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for test-user@KRBTEST.COM; defaulting to no policy
14:19:58.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:19:58.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "test-user@KRBTEST.COM" created.
14:19:58.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for joe-interloper@KRBTEST.COM; defaulting to no policy
14:19:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:19:58.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "joe-interloper@KRBTEST.COM" created.
14:19:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal slave/admin@KRBTEST.COM with password.
14:19:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/test-user.keytab.
14:19:58.871 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal test-user with kvno 1, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/test-user.keytab.
14:19:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:19:58 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:19:58.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:19:58.914 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-01T14:19:58Z chronyd version 4.6.1 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -NTS -SECHASH -IPV6 +DEBUG)
14:19:58.915 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-01T14:19:58Z Disabled control of system clock
14:19:58.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.213.254@KRBTEST.COM; defaulting to no policy
14:19:58.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:19:58.940 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.213.254@KRBTEST.COM" created.
14:19:58.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:19:58.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:19:58.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:19:58.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.13.213.254@KRBTEST.COM; defaulting to no policy
14:19:58.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:19:58.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.13.213.254@KRBTEST.COM" created.
14:19:59.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:19:59.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.254 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:19:59.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.254 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:19:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:59.003286 14167 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:19:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal
14:19:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data
14:19:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:19:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:19:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:19:59.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/logs
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data/info.pb
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:19:59.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.254:36127
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.254
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.254
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:19:59.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:19:59.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.301753 14197 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:19:59.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.302309 14197 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:19:59.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.302920 14197 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:19:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.335860 14197 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:19:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.336100 14197 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:19:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.336362 14197 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:19:59.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.336580 14197 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:19:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.336786 14197 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:19:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.337050 14197 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:19:59.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.337206 14197 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41679/jwks.json
14:19:59.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:59.374307 14197 master_runner.cc:386] Master server non-default flags:
14:19:59.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:19:59.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:19:59.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:19:59.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:19:59.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data
14:19:59.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal
14:19:59.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:19:59.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.254
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.254:36127
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data/info.pb
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.254
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:19:59.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/logs
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:59.375619 14197 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:19:59.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:19:59.376539 14197 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:19:59.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.386754 14203 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:00.790894 14202 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 14197
14:20:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.138041 14202 kernel_stack_watchdog.cc:198] Thread 14197 stuck at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:642 for 401ms:
14:20:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:20:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:20:01.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:20:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:20:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:19:59.387224 14204 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:01.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.139315 14197 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.752s user 0.655s sys 1.067s
14:20:01.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.139816 14197 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.753s user 0.656s sys 1.067s
14:20:01.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.142089 14206 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:01.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.144243 14205 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1756 milliseconds
14:20:01.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.144258 14197 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:20:01.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.145457 14197 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:01.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.147703 14197 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.149016 14197 hybrid_clock.cc:648] HybridClock initialized: now 1756736401148993 us; error 38 us; skew 500 ppm
14:20:01.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.149737 14197 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:01.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:01 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736401, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:01.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.158398 14197 init.cc:377] Logged in from keytab as kudu/127.13.213.254@KRBTEST.COM (short username kudu)
14:20:01.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.163020 14197 webserver.cc:480] Webserver started at http://127.13.213.254:38441/ using document root <none> and password file <none>
14:20:01.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.163978 14197 fs_manager.cc:362] Metadata directory not provided
14:20:01.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.164217 14197 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:01.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.164701 14197 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:20:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.169042 14197 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data/instance:
14:20:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e92c364bd674ed7b7de379369369941"
14:20:01.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:01 on dist-test-slave-k2rc"
14:20:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.170224 14197 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal/instance:
14:20:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e92c364bd674ed7b7de379369369941"
14:20:01.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:01 on dist-test-slave-k2rc"
14:20:01.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.177258 14197 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
14:20:01.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.182458 14214 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:01.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.183441 14197 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.005s sys 0.000s
14:20:01.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.183792 14197 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal
14:20:01.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e92c364bd674ed7b7de379369369941"
14:20:01.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:01 on dist-test-slave-k2rc"
14:20:01.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.184098 14197 fs_report.cc:389] FS layout report
14:20:01.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:01.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal
14:20:01.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data/data
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:01.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:01.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.248000 14197 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:01.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.253619 14197 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:01.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.254077 14197 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:01.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.336735 14197 rpc_server.cc:307] RPC server started. Bound to: 127.13.213.254:36127
14:20:01.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.336802 14265 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.213.254:36127 every 8 connection(s)
14:20:01.340 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.340577 14197 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data/info.pb
14:20:01.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.347126 14266 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:01.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.348093 14167 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 14197
14:20:01.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.348464 14167 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal/instance
14:20:01.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.359288 14266 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } has no permanent_uuid. Determining permanent_uuid...
14:20:01.372 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.213.253@KRBTEST.COM; defaulting to no policy
14:20:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:01.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.213.253@KRBTEST.COM" created.
14:20:01.390 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:01 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736401, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.254@KRBTEST.COM for kudu/127.13.213.254@KRBTEST.COM
14:20:01.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:01.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:01.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:01.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.403453 14266 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } has no permanent_uuid. Determining permanent_uuid...
14:20:01.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.405891 14218 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.213.253:36059: connect: Connection refused (error 111)
14:20:01.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.410223 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.253:36059: Network error: Client connection negotiation failed: client connection to 127.13.213.253:36059: connect: Connection refused (error 111)
14:20:01.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.13.213.253@KRBTEST.COM; defaulting to no policy
14:20:01.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:01.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.13.213.253@KRBTEST.COM" created.
14:20:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.253 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:01.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.253 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.441530 14167 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:01.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/logs
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data/info.pb
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:01.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.253:36059
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.253
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.253
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:20:01.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:20:01.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.459760 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } attempt: 1
14:20:01.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.463526 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.253:36059: Network error: Client connection negotiation failed: client connection to 127.13.213.253:36059: connect: Connection refused (error 111)
14:20:01.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.531944 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } attempt: 2
14:20:01.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.536374 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.253:36059: Network error: Client connection negotiation failed: client connection to 127.13.213.253:36059: connect: Connection refused (error 111)
14:20:01.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.627774 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } attempt: 3
14:20:01.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.631632 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.253:36059: Network error: Client connection negotiation failed: client connection to 127.13.213.253:36059: connect: Connection refused (error 111)
14:20:01.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.741748 14275 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:01.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.742283 14275 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:01.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.742870 14275 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:01.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.774514 14275 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:20:01.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.774783 14275 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:20:01.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.775200 14275 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:01.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.775080 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } attempt: 4
14:20:01.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.775524 14275 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:20:01.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.775826 14275 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:20:01.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.776041 14275 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:20:01.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.776176 14275 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41679/jwks.json
14:20:01.779 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.779311 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.253:36059: Network error: Client connection negotiation failed: client connection to 127.13.213.253:36059: connect: Connection refused (error 111)
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.813629 14275 master_runner.cc:386] Master server non-default flags:
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.253
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.253:36059
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data/info.pb
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.253
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:01.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/logs
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:01.815 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.815100 14275 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:01.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:01.816391 14275 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:01.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.827252 14281 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:02.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:02.078684 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } attempt: 5
14:20:02.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:02.085134 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.253:36059: Network error: Client connection negotiation failed: client connection to 127.13.213.253:36059: connect: Connection refused (error 111)
14:20:02.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:02.632618 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } attempt: 6
14:20:02.674 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:02.672737 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.253:36059: Network error: Client connection negotiation failed: client connection to 127.13.213.253:36059: connect: Connection refused (error 111)
14:20:03.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.231069 14280 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 14275
14:20:03.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.513152 14275 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.685s user 0.532s sys 1.071s
14:20:03.513 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.513567 14275 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.686s user 0.532s sys 1.072s
14:20:03.514 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:01.828094 14282 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:03.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.514607 14283 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1685 milliseconds
14:20:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.515857 14275 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:20:03.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.515921 14284 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:03.519 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.519683 14275 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:03.522 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.522161 14275 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:03.523 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.523494 14275 hybrid_clock.cc:648] HybridClock initialized: now 1756736403523468 us; error 35 us; skew 500 ppm
14:20:03.524 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.524255 14275 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:03.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:03 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736403, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:03.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.530076 14275 init.cc:377] Logged in from keytab as kudu/127.13.213.253@KRBTEST.COM (short username kudu)
14:20:03.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.534907 14275 webserver.cc:480] Webserver started at http://127.13.213.253:33719/ using document root <none> and password file <none>
14:20:03.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.535727 14275 fs_manager.cc:362] Metadata directory not provided
14:20:03.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.535907 14275 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:03.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.536283 14275 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:20:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.540371 14275 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data/instance:
14:20:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a4dc60a14a64f39a707a71d81e6a0a6"
14:20:03.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:03 on dist-test-slave-k2rc"
14:20:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.541416 14275 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal/instance:
14:20:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a4dc60a14a64f39a707a71d81e6a0a6"
14:20:03.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:03 on dist-test-slave-k2rc"
14:20:03.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.548180 14275 fs_manager.cc:696] Time spent creating directory manager: real 0.006s user 0.006s sys 0.000s
14:20:03.553 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.553439 14299 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:03.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.554473 14275 fs_manager.cc:730] Time spent opening block manager: real 0.003s user 0.003s sys 0.002s
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.554796 14275 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a4dc60a14a64f39a707a71d81e6a0a6"
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:03 on dist-test-slave-k2rc"
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.555128 14275 fs_report.cc:389] FS layout report
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data/data
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:03.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:03.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.604530 14275 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.618690 14275 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:03.619 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.619105 14275 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.690003 14275 rpc_server.cc:307] RPC server started. Bound to: 127.13.213.253:36059
14:20:03.690 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.690150 14350 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.213.253:36059 every 8 connection(s)
14:20:03.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.693228 14275 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data/info.pb
14:20:03.698 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.698544 14351 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.702008 14167 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 14275
14:20:03.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.702337 14167 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal/instance
14:20:03.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.709452 14351 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } has no permanent_uuid. Determining permanent_uuid...
14:20:03.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.213.252@KRBTEST.COM; defaulting to no policy
14:20:03.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:03.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.213.252@KRBTEST.COM" created.
14:20:03.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:03 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736403, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.253@KRBTEST.COM for kudu/127.13.213.254@KRBTEST.COM
14:20:03.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.733314 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } attempt: 7
14:20:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:03.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:03.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.747975 14351 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } has no permanent_uuid. Determining permanent_uuid...
14:20:03.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:03 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736401, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.254@KRBTEST.COM for kudu/127.13.213.253@KRBTEST.COM
14:20:03.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:03 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736403, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.253@KRBTEST.COM for kudu/127.13.213.253@KRBTEST.COM
14:20:03.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.13.213.252@KRBTEST.COM; defaulting to no policy
14:20:03.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.767654 14266 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } has no permanent_uuid. Determining permanent_uuid...
14:20:03.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:03.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.13.213.252@KRBTEST.COM" created.
14:20:03.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.773968 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:03.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.777714 14351 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } has no permanent_uuid. Determining permanent_uuid...
14:20:03.780 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.779824 14300 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:03.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.783921 14351 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:03.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:03.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.252 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:03.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.252 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.794190 14167 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal
14:20:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data
14:20:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:03.794 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/logs
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data/info.pb
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:03.795 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.252:33645
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.252
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.252
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:20:03.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:20:03.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.832345 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 1
14:20:03.833 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.833494 14351 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 1
14:20:03.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.836809 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:03.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.837291 14351 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:03.906 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.905706 14351 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 2
14:20:03.910 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.910053 14351 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:03.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:03.918260 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 2
14:20:03.922 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:03.922413 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:04.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:04.001490 14351 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 3
14:20:04.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.005296 14351 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:04.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:04.007781 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 3
14:20:04.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.011849 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:04.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.113240 14364 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:04.113 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.113794 14364 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:04.114 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.114405 14364 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.146011 14364 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:20:04.146 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.146384 14364 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:20:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.146880 14364 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.147166 14364 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:20:04.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.147373 14364 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:20:04.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.147567 14364 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:20:04.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.147735 14364 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41679/jwks.json
14:20:04.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:04.148765 14351 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 4
14:20:04.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:04.152287 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 4
14:20:04.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.152861 14351 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:04.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.155917 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:04.186254 14364 master_runner.cc:386] Master server non-default flags:
14:20:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data
14:20:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal
14:20:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:20:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:04.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.252
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.252:33645
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data/info.pb
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.252
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/logs
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:04.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:04.187603 14364 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:04.189 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:04.188942 14364 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:04.203 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.202507 14373 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:04.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:04.439302 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 5
14:20:04.445 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.445439 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:04.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:04.452306 14351 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 5
14:20:04.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.459513 14351 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:05.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:04.999969 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 6
14:20:05.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.027093 14351 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 6
14:20:05.038 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:05.037668 14266 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:05.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:05.038507 14351 consensus_peers.cc:648] Error getting permanent uuid from config peer 127.13.213.252:33645: Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:05.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:04.205257 14374 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:05.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:05.533187 14376 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:05.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:05.535295 14364 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.329s user 0.000s sys 0.011s
14:20:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:05.535810 14364 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.330s user 0.000s sys 0.013s
14:20:05.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.536085 14364 server_base.cc:1047] running on GCE node
14:20:05.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.537210 14364 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:05.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.541375 14364 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:05.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.542704 14364 hybrid_clock.cc:648] HybridClock initialized: now 1756736405542657 us; error 55 us; skew 500 ppm
14:20:05.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.543578 14364 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:05.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:05 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736405, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:05.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.550379 14364 init.cc:377] Logged in from keytab as kudu/127.13.213.252@KRBTEST.COM (short username kudu)
14:20:05.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.556406 14364 webserver.cc:480] Webserver started at http://127.13.213.252:43179/ using document root <none> and password file <none>
14:20:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.557293 14364 fs_manager.cc:362] Metadata directory not provided
14:20:05.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.557471 14364 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:05.558 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.557881 14364 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:20:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.562273 14364 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data/instance:
14:20:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4edc07c774c448db8853b5fdb7bbe6e9"
14:20:05.562 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:05 on dist-test-slave-k2rc"
14:20:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.563566 14364 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal/instance:
14:20:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4edc07c774c448db8853b5fdb7bbe6e9"
14:20:05.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:05 on dist-test-slave-k2rc"
14:20:05.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.571611 14364 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.009s sys 0.000s
14:20:05.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.577876 14389 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.578886 14364 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.002s sys 0.002s
14:20:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.579165 14364 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal
14:20:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4edc07c774c448db8853b5fdb7bbe6e9"
14:20:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:05 on dist-test-slave-k2rc"
14:20:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.579448 14364 fs_report.cc:389] FS layout report
14:20:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:05.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data/data
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:05.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:05.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:05.648283 14347 debug-util.cc:398] Leaking SignalData structure 0x7b0800071060 after lost signal to thread 14276
14:20:05.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.648473 14364 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:05.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:05.648968 14347 debug-util.cc:398] Leaking SignalData structure 0x7b08000717c0 after lost signal to thread 14350
14:20:05.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.656369 14364 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:05.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.656989 14364 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:05.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.746351 14364 rpc_server.cc:307] RPC server started. Bound to: 127.13.213.252:33645
14:20:05.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.746492 14440 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.213.252:33645 every 8 connection(s)
14:20:05.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.749223 14364 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data/info.pb
14:20:05.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.755528 14441 data_dirs.cc:400] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
14:20:05.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.758315 14167 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 14364
14:20:05.759 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.758751 14167 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal/instance
14:20:05.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.772715 14441 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } has no permanent_uuid. Determining permanent_uuid...
14:20:05.802 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:05 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.213.254@KRBTEST.COM
14:20:05.803 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:05 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736405, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.252@KRBTEST.COM for kudu/127.13.213.254@KRBTEST.COM
14:20:05.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.821772 14441 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } has no permanent_uuid. Determining permanent_uuid...
14:20:05.830 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:05 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.213.253@KRBTEST.COM
14:20:05.852 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:05 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736398, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for kudu/127.13.213.252@KRBTEST.COM
14:20:05.855 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:05 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736405, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.252@KRBTEST.COM for kudu/127.13.213.253@KRBTEST.COM
14:20:05.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.873273 14441 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } has no permanent_uuid. Determining permanent_uuid...
14:20:05.886 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:05 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736405, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.252@KRBTEST.COM for kudu/127.13.213.252@KRBTEST.COM
14:20:05.911 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.910521 14441 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: Bootstrap starting.
14:20:05.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.918601 14441 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: Neither blocks nor log segments found. Creating new log.
14:20:05.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.920966 14441 log.cc:826] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: Log is configured to *not* fsync() on all Append() calls
14:20:05.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.926436 14441 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: No bootstrap required, opened a new log
14:20:05.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.954795 14441 raft_consensus.cc:357] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:05.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.955742 14441 raft_consensus.cc:383] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:05.956 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.956063 14441 raft_consensus.cc:738] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4edc07c774c448db8853b5fdb7bbe6e9, State: Initialized, Role: FOLLOWER
14:20:05.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.957113 14441 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:05.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.960623 14451 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:05.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.961247 14451 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: This master's current role is: FOLLOWER
14:20:05.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.962863 14441 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: configured and running, proceeding with master startup.
14:20:05.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:05.996843 14462 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:20:05.997 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:05.997466 14462 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:20:06.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:05.990480 14364 master_runner.cc:186] Error getting master registration for 127.13.213.254:36127: OK, instance_id { permanent_uuid: "9e92c364bd674ed7b7de379369369941" instance_seqno: 1756736401313144 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:20:06.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.014344 14364 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:20:06.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.071101 14266 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 7
14:20:06.094 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:06 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736401, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.254@KRBTEST.COM for kudu/127.13.213.252@KRBTEST.COM
14:20:06.100 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.099414 14351 consensus_peers.cc:658] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } attempt: 7
14:20:06.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.124220 14266 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: Bootstrap starting.
14:20:06.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:06 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736403, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.253@KRBTEST.COM for kudu/127.13.213.252@KRBTEST.COM
14:20:06.132 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.132613 14266 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: Neither blocks nor log segments found. Creating new log.
14:20:06.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.153146 14266 log.cc:826] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: Log is configured to *not* fsync() on all Append() calls
14:20:06.164 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:06.163851 14395 tablet.cc:2378] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:20:06.165 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.165596 14351 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: Bootstrap starting.
14:20:06.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.168362 14266 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: No bootstrap required, opened a new log
14:20:06.175 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.174996 14351 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: Neither blocks nor log segments found. Creating new log.
14:20:06.177 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.177515 14351 log.cc:826] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: Log is configured to *not* fsync() on all Append() calls
14:20:06.191 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.190626 14351 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: No bootstrap required, opened a new log
14:20:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.207347 14266 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:06.208 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.208523 14266 raft_consensus.cc:383] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:06.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.209002 14266 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e92c364bd674ed7b7de379369369941, State: Initialized, Role: FOLLOWER
14:20:06.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.210054 14266 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:06.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.226047 14469 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:06.227 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.227126 14469 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: This master's current role is: FOLLOWER
14:20:06.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.228694 14266 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: configured and running, proceeding with master startup.
14:20:06.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:06.268946 14220 tablet.cc:2378] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:20:06.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.290411 14351 raft_consensus.cc:357] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:06.301 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.301447 14351 raft_consensus.cc:383] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
14:20:06.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.302001 14351 raft_consensus.cc:738] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a4dc60a14a64f39a707a71d81e6a0a6, State: Initialized, Role: FOLLOWER
14:20:06.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.313325 14351 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:06.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.331154 14351 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: configured and running, proceeding with master startup.
14:20:06.332 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.331954 14451 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:20:06.333 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.332455 14451 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:06.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.336544 14451 leader_election.cc:290] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 9e92c364bd674ed7b7de379369369941 (127.13.213.254:36127), 2a4dc60a14a64f39a707a71d81e6a0a6 (127.13.213.253:36059)
14:20:06.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.336606 14241 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e92c364bd674ed7b7de379369369941" is_pre_election: true
14:20:06.338 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.337407 14241 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4edc07c774c448db8853b5fdb7bbe6e9 in term 0.
14:20:06.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.343497 14391 leader_election.cc:304] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4edc07c774c448db8853b5fdb7bbe6e9, 9e92c364bd674ed7b7de379369369941; no voters:
14:20:06.344 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.344375 14451 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 0 FOLLOWER]: Leader pre-election won for term 1
14:20:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.344738 14451 raft_consensus.cc:491] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:20:06.345 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.345118 14451 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 0 FOLLOWER]: Advancing to term 1
14:20:06.353 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.352794 14451 raft_consensus.cc:513] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:06.362 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.356107 14451 leader_election.cc:290] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [CANDIDATE]: Term 1 election: Requested vote from peers 9e92c364bd674ed7b7de379369369941 (127.13.213.254:36127), 2a4dc60a14a64f39a707a71d81e6a0a6 (127.13.213.253:36059)
14:20:06.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.356138 14241 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "9e92c364bd674ed7b7de379369369941"
14:20:06.363 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.356640 14241 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [term 0 FOLLOWER]: Advancing to term 1
14:20:06.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.365594 14325 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6"
14:20:06.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.366642 14325 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 0 FOLLOWER]: Advancing to term 1
14:20:06.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.345003 14326 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" is_pre_election: true
14:20:06.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.327941 14477 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:06.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.377192 14477 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: This master's current role is: FOLLOWER
14:20:06.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.378746 14325 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4edc07c774c448db8853b5fdb7bbe6e9 in term 1.
14:20:06.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.380211 14393 leader_election.cc:304] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2a4dc60a14a64f39a707a71d81e6a0a6, 4edc07c774c448db8853b5fdb7bbe6e9; no voters:
14:20:06.382 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.381138 14451 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 1 FOLLOWER]: Leader election won for term 1
14:20:06.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.385525 14241 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4edc07c774c448db8853b5fdb7bbe6e9 in term 1.
14:20:06.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.400059 14451 raft_consensus.cc:695] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 1 LEADER]: Becoming Leader. State: Replica: 4edc07c774c448db8853b5fdb7bbe6e9, State: Running, Role: LEADER
14:20:06.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.401361 14451 consensus_queue.cc:237] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:06.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:06.425653 14305 tablet.cc:2378] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: Can't schedule compaction. Clean time has not been advanced past its initial value.
14:20:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.452459 14483 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4edc07c774c448db8853b5fdb7bbe6e9. Latest consensus state: current_term: 1 leader_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:06.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.453553 14483 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: This master's current role is: LEADER
14:20:06.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.465966 14495 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:20:06.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:06.467939 14485 catalog_manager.cc:1560] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
14:20:06.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.469379 14497 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:20:06.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:06.470793 14485 catalog_manager.cc:875] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
14:20:06.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.414947 14197 master_runner.cc:186] Error getting master registration for 127.13.213.253:36059: OK, instance_id { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" instance_seqno: 1756736403669079 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
14:20:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.502050 14197 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:20:06.502 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.502570 14495 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:20:06.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.543864 14325 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Refusing update from remote peer 4edc07c774c448db8853b5fdb7bbe6e9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:20:06.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.546283 14451 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
14:20:06.549 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.548712 14241 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [term 1 FOLLOWER]: Refusing update from remote peer 4edc07c774c448db8853b5fdb7bbe6e9: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
14:20:06.551 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.550347 14451 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [LEADER]: Connected to new peer: Peer: permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
14:20:06.615 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.614300 14469 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4edc07c774c448db8853b5fdb7bbe6e9. Latest consensus state: current_term: 1 leader_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:06.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.615841 14469 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: This master's current role is: FOLLOWER
14:20:06.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.628577 14500 mvcc.cc:204] Tried to move back new op lower bound from 7195592321140146176 to 7195592320635224064. Current Snapshot: MvccSnapshot[applied={T|T < 7195592321140146176}]
14:20:06.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.636009 14477 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 4edc07c774c448db8853b5fdb7bbe6e9. Latest consensus state: current_term: 1 leader_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:06.638 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.638222 14477 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: This master's current role is: FOLLOWER
14:20:06.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.652813 14469 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:06.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.653689 14469 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: This master's current role is: FOLLOWER
14:20:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.663022 14477 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:06.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.663662 14477 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: This master's current role is: FOLLOWER
14:20:06.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.666132 14451 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:06.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.666819 14451 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: This master's current role is: LEADER
14:20:06.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.667382 14451 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:06.669 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.667979 14451 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: This master's current role is: LEADER
14:20:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.687563 14495 catalog_manager.cc:1349] Generated new cluster ID: fa90510be2dc4b91a9cfe573748fb8f2
14:20:06.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.687870 14495 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:20:06.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.772719 14495 catalog_manager.cc:1372] Generated new certificate authority record
14:20:06.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.775216 14495 catalog_manager.cc:1506] Loading token signing keys...
14:20:06.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.830708 14495 catalog_manager.cc:5955] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: Generated new TSK 0
14:20:06.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:06.832456 14495 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:20:06.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.213.193@KRBTEST.COM; defaulting to no policy
14:20:06.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:06.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.213.193@KRBTEST.COM" created.
14:20:06.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:06.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:06.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:06.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.13.213.193@KRBTEST.COM; defaulting to no policy
14:20:06.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:06.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.13.213.193@KRBTEST.COM" created.
14:20:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.193 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:07.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.193 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.019534 14167 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/wal
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/data
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/logs
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/data/info.pb
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:07.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.193:0
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.213.193
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.193
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.193
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:20:07.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:20:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:07.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:20:07.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.476557 14485 catalog_manager.cc:1261] Loaded cluster ID: fa90510be2dc4b91a9cfe573748fb8f2
14:20:07.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.478029 14485 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: loading cluster ID for follower catalog manager: success
14:20:07.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.485368 14485 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: acquiring CA information for follower catalog manager: success
14:20:07.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.490839 14485 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:20:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.509075 14497 catalog_manager.cc:1261] Loaded cluster ID: fa90510be2dc4b91a9cfe573748fb8f2
14:20:07.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.509568 14497 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: loading cluster ID for follower catalog manager: success
14:20:07.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.516803 14497 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: acquiring CA information for follower catalog manager: success
14:20:07.544 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.544476 14497 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:20:07.622 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:07.622044 14512 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:07.623 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:07.622865 14512 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:07.624 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:07.623926 14512 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:07.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:07.692204 14512 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:07.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:07.693926 14512 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.213.193
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.773803 14512 tablet_server_runner.cc:78] Tablet server non-default flags:
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/data
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/wal
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.193
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.193:0
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/data/info.pb
14:20:07.774 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.193
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.213.193
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/logs
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:20:07.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:07.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.775826 14512 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:07.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:07.777164 14512 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:07.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:07.818555 14519 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:10.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:09.199271 14518 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 14512
14:20:10.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:10.436611 14512 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.614s user 0.788s sys 1.252s
14:20:10.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:10.436954 14512 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.615s user 0.788s sys 1.252s
14:20:10.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:07.823987 14520 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:10.453 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.453186 14512 server_base.cc:1047] running on GCE node
14:20:10.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:10.455017 14526 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:10.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.456216 14512 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:10.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.459158 14512 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:10.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.460691 14512 hybrid_clock.cc:648] HybridClock initialized: now 1756736410460650 us; error 40 us; skew 500 ppm
14:20:10.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.461870 14512 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:10.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:10 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736410, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:10.469 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.469054 14512 init.cc:377] Logged in from keytab as kudu/127.13.213.193@KRBTEST.COM (short username kudu)
14:20:10.475 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.475232 14512 webserver.cc:480] Webserver started at http://127.13.213.193:44903/ using document root <none> and password file <none>
14:20:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.476222 14512 fs_manager.cc:362] Metadata directory not provided
14:20:10.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.476436 14512 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:10.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.476954 14512 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:20:10.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.481210 14512 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/data/instance:
14:20:10.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ada331dc71b4be3978332266014431f"
14:20:10.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:10 on dist-test-slave-k2rc"
14:20:10.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.482304 14512 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/wal/instance:
14:20:10.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ada331dc71b4be3978332266014431f"
14:20:10.482 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:10 on dist-test-slave-k2rc"
14:20:10.490 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.490322 14512 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
14:20:10.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.496155 14532 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:10.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.497295 14512 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:20:10.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.497628 14512 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/wal
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "1ada331dc71b4be3978332266014431f"
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:10 on dist-test-slave-k2rc"
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.497967 14512 fs_report.cc:389] FS layout report
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/wal
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/wal
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/data/data
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:10.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:10.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:10.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:10.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:10.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:10.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.546695 14512 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.555636 14512 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:10.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.556038 14512 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:10.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.559489 14512 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:20:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.565011 14512 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:20:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.565253 14512 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:20:10.565 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.565632 14512 ts_tablet_manager.cc:610] Registered 0 tablets
14:20:10.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.565809 14512 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:20:10.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.743624 14512 rpc_server.cc:307] RPC server started. Bound to: 127.13.213.193:40897
14:20:10.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.743747 14644 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.213.193:40897 every 8 connection(s)
14:20:10.747 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.747125 14512 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/data/info.pb
14:20:10.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.752526 14167 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 14512
14:20:10.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.753046 14167 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-0/wal/instance
14:20:10.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.213.194@KRBTEST.COM; defaulting to no policy
14:20:10.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:10 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736410, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.193@KRBTEST.COM for kudu/127.13.213.252@KRBTEST.COM
14:20:10.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:10 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736410, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.193@KRBTEST.COM for kudu/127.13.213.253@KRBTEST.COM
14:20:10.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:10.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.213.194@KRBTEST.COM" created.
14:20:10.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.858510 14645 heartbeater.cc:344] Connected to a master server at 127.13.213.252:33645
14:20:10.861 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.859202 14645 heartbeater.cc:461] Registering TS with master...
14:20:10.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.861493 14645 heartbeater.cc:507] Master 127.13.213.252:33645 requested a full tablet report, sending...
14:20:10.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.865643 14406 ts_manager.cc:194] Registered new tserver with Master: 1ada331dc71b4be3978332266014431f (127.13.213.193:40897)
14:20:10.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.869243 14406 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.13.213.193@KRBTEST.COM'} at 127.13.213.193:55129
14:20:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.902750 14646 heartbeater.cc:344] Connected to a master server at 127.13.213.253:36059
14:20:10.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.903159 14646 heartbeater.cc:461] Registering TS with master...
14:20:10.904 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.904000 14646 heartbeater.cc:507] Master 127.13.213.253:36059 requested a full tablet report, sending...
14:20:10.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.907496 14316 ts_manager.cc:194] Registered new tserver with Master: 1ada331dc71b4be3978332266014431f (127.13.213.193:40897)
14:20:10.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:10 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736410, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.193@KRBTEST.COM for kudu/127.13.213.254@KRBTEST.COM
14:20:10.918 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:10.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:10.919 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:10.920 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:10.919652 14167 mini_kdc.cc:279] Time spent creating service keytab for kudu/127.13.213.194: real 0.153s user 0.006s sys 0.009s
14:20:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.931162 14650 heartbeater.cc:344] Connected to a master server at 127.13.213.254:36127
14:20:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.931564 14650 heartbeater.cc:461] Registering TS with master...
14:20:10.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.932233 14650 heartbeater.cc:507] Master 127.13.213.254:36127 requested a full tablet report, sending...
14:20:10.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.935304 14231 ts_manager.cc:194] Registered new tserver with Master: 1ada331dc71b4be3978332266014431f (127.13.213.193:40897)
14:20:10.938 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.13.213.194@KRBTEST.COM; defaulting to no policy
14:20:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:10.941 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.13.213.194@KRBTEST.COM" created.
14:20:10.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:10.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.194 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:10.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.194 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:10.965 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:10.965536 14167 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/wal
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/data
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/logs
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/data/info.pb
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.194:0
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.213.194
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.194
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:10.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.194
14:20:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:20:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:20:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:20:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:10.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:20:11.162 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:11 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736411, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:11.163 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:11.163074 14210 init.cc:295] Successfully reacquired a new kerberos TGT
14:20:11.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:11.301862 14659 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:11.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:11.302322 14659 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:11.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:11.302892 14659 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:11.336 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:11.335700 14659 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:11.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:11.336906 14659 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.213.194
14:20:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:11.376286 14659 tablet_server_runner.cc:78] Tablet server non-default flags:
14:20:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:11.376 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/data
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/wal
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.194
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.194:0
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:20:11.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/data/info.pb
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.194
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.213.194
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/logs
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:20:11.378 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:20:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:20:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:20:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:11.378028 14659 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:11.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:11.379417 14659 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:11.398 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:11.394716 14665 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:11.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:11.875166 14645 heartbeater.cc:499] Master 127.13.213.252:33645 was elected leader, sending a full tablet report...
14:20:13.455 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:11.402812 14666 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:13.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:13.455724 14659 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.054s user 0.691s sys 1.191s
14:20:13.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:13.456355 14659 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.055s user 0.691s sys 1.191s
14:20:13.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:12.795687 14664 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 14659
14:20:13.458 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:13.458642 14668 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:13.462738 14667 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 2060 milliseconds
14:20:13.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.462774 14659 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:20:13.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.463927 14659 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:13.466 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.466110 14659 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:13.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.467475 14659 hybrid_clock.cc:648] HybridClock initialized: now 1756736413467430 us; error 47 us; skew 500 ppm
14:20:13.468 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.468350 14659 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:13.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:13 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736413, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:13.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.474287 14659 init.cc:377] Logged in from keytab as kudu/127.13.213.194@KRBTEST.COM (short username kudu)
14:20:13.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.478793 14659 webserver.cc:480] Webserver started at http://127.13.213.194:37409/ using document root <none> and password file <none>
14:20:13.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.479628 14659 fs_manager.cc:362] Metadata directory not provided
14:20:13.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.479804 14659 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:13.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.480176 14659 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:20:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.484792 14659 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/data/instance:
14:20:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "097ba40d6e064c87afbe3cb2ad79f915"
14:20:13.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:13 on dist-test-slave-k2rc"
14:20:13.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.485946 14659 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/wal/instance:
14:20:13.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "097ba40d6e064c87afbe3cb2ad79f915"
14:20:13.486 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:13 on dist-test-slave-k2rc"
14:20:13.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.493414 14659 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.007s sys 0.000s
14:20:13.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.498968 14680 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.500074 14659 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.001s
14:20:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.500344 14659 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/wal
14:20:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "097ba40d6e064c87afbe3cb2ad79f915"
14:20:13.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:13 on dist-test-slave-k2rc"
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.500630 14659 fs_report.cc:389] FS layout report
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/wal
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/wal
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/data/data
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:13.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:13.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:13 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736413, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:13.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.534515 14295 init.cc:295] Successfully reacquired a new kerberos TGT
14:20:13.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.561077 14659 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:13.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.567252 14659 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:13.568 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.567785 14659 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:13.570 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.570240 14659 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:20:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.574141 14659 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:20:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.574331 14659 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
14:20:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.574523 14659 ts_tablet_manager.cc:610] Registered 0 tablets
14:20:13.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.574653 14659 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:20:13.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.710480 14659 rpc_server.cc:307] RPC server started. Bound to: 127.13.213.194:41985
14:20:13.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.710597 14792 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.213.194:41985 every 8 connection(s)
14:20:13.713 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.713120 14659 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/data/info.pb
14:20:13.724 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.722472 14167 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 14659
14:20:13.725 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.722980 14167 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-1/wal/instance
14:20:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:13 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736413, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.194@KRBTEST.COM for kudu/127.13.213.253@KRBTEST.COM
14:20:13.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:13 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736413, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.194@KRBTEST.COM for kudu/127.13.213.252@KRBTEST.COM
14:20:13.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.803802 14794 heartbeater.cc:344] Connected to a master server at 127.13.213.253:36059
14:20:13.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.804373 14794 heartbeater.cc:461] Registering TS with master...
14:20:13.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.805646 14794 heartbeater.cc:507] Master 127.13.213.253:36059 requested a full tablet report, sending...
14:20:13.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.808907 14316 ts_manager.cc:194] Registered new tserver with Master: 097ba40d6e064c87afbe3cb2ad79f915 (127.13.213.194:41985)
14:20:13.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.813202 14793 heartbeater.cc:344] Connected to a master server at 127.13.213.252:33645
14:20:13.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.813568 14793 heartbeater.cc:461] Registering TS with master...
14:20:13.814 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.814600 14793 heartbeater.cc:507] Master 127.13.213.252:33645 requested a full tablet report, sending...
14:20:13.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.820267 14406 ts_manager.cc:194] Registered new tserver with Master: 097ba40d6e064c87afbe3cb2ad79f915 (127.13.213.194:41985)
14:20:13.822 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:13 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736413, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.194@KRBTEST.COM for kudu/127.13.213.254@KRBTEST.COM
14:20:13.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.822587 14406 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.13.213.194@KRBTEST.COM'} at 127.13.213.194:46987
14:20:13.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for kudu/127.13.213.195@KRBTEST.COM; defaulting to no policy
14:20:13.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:13.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "kudu/127.13.213.195@KRBTEST.COM" created.
14:20:13.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.838243 14795 heartbeater.cc:344] Connected to a master server at 127.13.213.254:36127
14:20:13.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.838591 14795 heartbeater.cc:461] Registering TS with master...
14:20:13.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.839233 14795 heartbeater.cc:507] Master 127.13.213.254:36127 requested a full tablet report, sending...
14:20:13.842 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.841910 14231 ts_manager.cc:194] Registered new tserver with Master: 097ba40d6e064c87afbe3cb2ad79f915 (127.13.213.194:41985)
14:20:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:13.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal kudu/127.13.213.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:13.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) WARNING: no policy specified for HTTP/127.13.213.195@KRBTEST.COM; defaulting to no policy
14:20:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:13.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Principal "HTTP/127.13.213.195@KRBTEST.COM" created.
14:20:13.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Authenticating as principal test-admin/admin@KRBTEST.COM with password.
14:20:13.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.195 with kvno 2, encryption type aes256-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:13.899 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Entry for principal HTTP/127.13.213.195 with kvno 2, encryption type aes128-cts-hmac-sha1-96 added to keytab WRFILE:/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab.
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:13.901028 14167 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/wal
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/data
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/logs
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/data/info.pb
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:13.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.195:0
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.213.195
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.195
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.195
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:13.902 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false with env {KRB5CCNAME=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:20:14.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:14.213207 14807 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:14.213701 14807 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:14.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:14.214560 14807 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:14.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:14.249586 14807 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:14.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:14.250663 14807 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.13.213.195
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:14.289920 14807 tablet_server_runner.cc:78] Tablet server non-default flags:
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/data
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/wal
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.195
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.195:0
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:20:14.290 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/data/info.pb
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.195
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.13.213.195
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/logs
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:14.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:14.291651 14807 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:14.292 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:14.292659 14807 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:14.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:14.304364 14815 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:14.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:14.827069 14793 heartbeater.cc:499] Master 127.13.213.252:33645 was elected leader, sending a full tablet report...
14:20:15.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:15 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736415, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:15.557 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.557384 14385 init.cc:295] Successfully reacquired a new kerberos TGT
14:20:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:15.710600 14814 debug-util.cc:398] Leaking SignalData structure 0x7b0800037b60 after lost signal to thread 14807
14:20:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:14.305284 14816 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:15.934909 14814 kernel_stack_watchdog.cc:198] Thread 14807 stuck at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:642 for 400ms:
14:20:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Kernel stack:
14:20:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) (could not read kernel stack)
14:20:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) User stack:
14:20:15.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) <Timed out: thread did not respond: maybe it is blocking signals>
14:20:15.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:15.935592 14807 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.630s user 0.525s sys 1.069s
14:20:15.936 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:15.936576 14807 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.631s user 0.525s sys 1.069s
14:20:15.937 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:15.937395 14818 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:15.942888 14817 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1636 milliseconds
14:20:15.943 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.942935 14807 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:20:15.944 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.944231 14807 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:15.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.946291 14807 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:15.947 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.947647 14807 hybrid_clock.cc:648] HybridClock initialized: now 1756736415947604 us; error 39 us; skew 500 ppm
14:20:15.948 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.948665 14807 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:15.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:15 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736415, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:15.955 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.954952 14807 init.cc:377] Logged in from keytab as kudu/127.13.213.195@KRBTEST.COM (short username kudu)
14:20:15.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.960038 14807 webserver.cc:480] Webserver started at http://127.13.213.195:38585/ using document root <none> and password file <none>
14:20:15.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.960899 14807 fs_manager.cc:362] Metadata directory not provided
14:20:15.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.961094 14807 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:15.961 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.961460 14807 server_base.cc:895] This appears to be a new deployment of Kudu; creating new FS layout
14:20:15.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.966156 14807 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/data/instance:
14:20:15.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a04c7b44f494e3e84a5f643a35bf3c5"
14:20:15.966 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:15 on dist-test-slave-k2rc"
14:20:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.967161 14807 fs_manager.cc:1068] Generated new instance metadata in path /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/wal/instance:
14:20:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a04c7b44f494e3e84a5f643a35bf3c5"
14:20:15.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:15 on dist-test-slave-k2rc"
14:20:15.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.974500 14807 fs_manager.cc:696] Time spent creating directory manager: real 0.007s user 0.008s sys 0.000s
14:20:15.980 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.980394 14827 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:15.981 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.981592 14807 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
14:20:15.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.981940 14807 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/wal
14:20:15.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "3a04c7b44f494e3e84a5f643a35bf3c5"
14:20:15.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:15 on dist-test-slave-k2rc"
14:20:15.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:15.982313 14807 fs_report.cc:389] FS layout report
14:20:15.982 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/wal
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/wal
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/data/data
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:15.983 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:16.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.053740 14807 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:16.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.065900 14807 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:16.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.066407 14807 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:16.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.071853 14807 txn_system_client.cc:432] TxnSystemClient initialization is disabled...
14:20:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.076869 14807 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
14:20:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.077093 14807 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
14:20:16.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.077404 14807 ts_tablet_manager.cc:610] Registered 0 tablets
14:20:16.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.077605 14807 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
14:20:16.307 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.307214 14807 rpc_server.cc:307] RPC server started. Bound to: 127.13.213.195:33201
14:20:16.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.307399 14940 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.213.195:33201 every 8 connection(s)
14:20:16.311 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.311220 14807 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/data/info.pb
14:20:16.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.314546 14167 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 14807
14:20:16.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.315026 14167 external_mini_cluster.cc:1442] Reading /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/ts-2/wal/instance
14:20:16.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:16 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736415, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.195@KRBTEST.COM for kudu/127.13.213.254@KRBTEST.COM
14:20:16.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:16 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736415, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.195@KRBTEST.COM for kudu/127.13.213.253@KRBTEST.COM
14:20:16.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:16 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736415, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.195@KRBTEST.COM for kudu/127.13.213.252@KRBTEST.COM
14:20:16.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.430943 14946 heartbeater.cc:344] Connected to a master server at 127.13.213.254:36127
14:20:16.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.431478 14946 heartbeater.cc:461] Registering TS with master...
14:20:16.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.433070 14946 heartbeater.cc:507] Master 127.13.213.254:36127 requested a full tablet report, sending...
14:20:16.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.433310 14941 heartbeater.cc:344] Connected to a master server at 127.13.213.252:33645
14:20:16.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.433687 14941 heartbeater.cc:461] Registering TS with master...
14:20:16.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.434759 14941 heartbeater.cc:507] Master 127.13.213.252:33645 requested a full tablet report, sending...
14:20:16.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.435352 14942 heartbeater.cc:344] Connected to a master server at 127.13.213.253:36059
14:20:16.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.435698 14942 heartbeater.cc:461] Registering TS with master...
14:20:16.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.436056 14231 ts_manager.cc:194] Registered new tserver with Master: 3a04c7b44f494e3e84a5f643a35bf3c5 (127.13.213.195:33201)
14:20:16.437 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.436630 14942 heartbeater.cc:507] Master 127.13.213.253:36059 requested a full tablet report, sending...
14:20:16.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.437307 14406 ts_manager.cc:194] Registered new tserver with Master: 3a04c7b44f494e3e84a5f643a35bf3c5 (127.13.213.195:33201)
14:20:16.439 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.439158 14316 ts_manager.cc:194] Registered new tserver with Master: 3a04c7b44f494e3e84a5f643a35bf3c5 (127.13.213.195:33201)
14:20:16.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.439749 14406 master_service.cc:496] Signed X509 certificate for tserver {username='kudu', principal='kudu/127.13.213.195@KRBTEST.COM'} at 127.13.213.195:46715
14:20:16.457 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.456769 14167 external_mini_cluster.cc:949] 3 TS(s) registered with all masters
14:20:16.469 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:20:16.470 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_kdc_env_vars {
}
14:20:16.480 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_kdc_env_vars {
env_vars {
key: "KRB5CCNAME"
value: "/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc"
}
env_vars {
key: "KRB5_CONFIG"
value: "/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5.conf"
}
env_vars {
key: "KRB5_KDC_PROFILE"
value: "/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kdc.conf"
}
env_vars {
key: "KUDU_ENABLE_KRB5_REALM_FIX"
value: "yes"
}
}
14:20:16.481 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_masters {
}
14:20:16.484 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.13.213.254"
port: 36127
}
bound_http_address {
host: "127.13.213.254"
port: 38441
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.13.213.253"
port: 36059
}
bound_http_address {
host: "127.13.213.253"
port: 33719
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.13.213.252"
port: 33645
}
bound_http_address {
host: "127.13.213.252"
port: 43179
}
}
}
14:20:16.485 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: get_tservers {
}
14:20:16.486 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.13.213.193"
port: 40897
}
bound_http_address {
host: "127.13.213.193"
port: 44903
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.13.213.194"
port: 41985
}
bound_http_address {
host: "127.13.213.194"
port: 37409
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.13.213.195"
port: 33201
}
bound_http_address {
host: "127.13.213.195"
port: 38585
}
}
}
14:20:16.486 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:20:16.488 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:20:16.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:16 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736416, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:16.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:20:16.521 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:20:16.523 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc
14:20:16.530 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:20:16.531 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc
14:20:16.533 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:20:16.536 [DEBUG - Time-limited test] (SecurityContext.java:158) Using caller-provided subject with Kerberos principal test-admin@KRBTEST.COM. Caller is responsible for refreshing credentials.
14:20:16.538 [INFO - Time-limited test] (MiniKuduCluster.java:622) Running kinit for user test-admin
14:20:16.538 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: kinit {
username: "test-admin"
}
14:20:16.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:16 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736416, etypes {rep=17 tkt=17 ses=17}, test-admin@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:16.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Password for test-admin@KRBTEST.COM:
14:20:16.568 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:20:16.569 [DEBUG - Time-limited test] (SecurityUtil.java:105) Using ticketCache: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc
14:20:16.570 [DEBUG - Time-limited test] (SecurityUtil.java:119) Logged in as subject: test-admin@KRBTEST.COM
14:20:16.571 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.213.253:36059
14:20:16.573 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
14:20:16.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.573695 14167 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 14275
14:20:16.602 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:20:16.602 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.213.252:33645
14:20:16.603 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
14:20:16.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.603437 14167 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 14364
14:20:16.630 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:20:16.631 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.13.213.254:36127
14:20:16.631 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
14:20:16.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.631629 14167 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 14197
14:20:16.660 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:20:16.660 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.213.253:36059
14:20:16.664 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
14:20:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:16.666291 14167 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal
14:20:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data
14:20:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:16.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/logs
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data/info.pb
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.253:36059
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.253
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33719
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.253
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:20:16.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:20:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:20:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:20:16.668 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:20:16.823 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:16.823304 14794 heartbeater.cc:646] Failed to heartbeat to 127.13.213.253:36059 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.13.213.253:36059: connect: Connection refused (error 111)
14:20:16.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:16.899760 14645 heartbeater.cc:646] Failed to heartbeat to 127.13.213.252:33645 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:16.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:16.990815 14957 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:16.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:16.991374 14957 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:16.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:16.992087 14957 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.028488 14957 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:20:17.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.028735 14957 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:20:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.029070 14957 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.029292 14957 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:20:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.029493 14957 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:20:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.029683 14957 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:20:17.029 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.029840 14957 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41679/jwks.json
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:17.070724 14957 master_runner.cc:386] Master server non-default flags:
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.253
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.253:36059
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:17.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data/info.pb
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.253
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=33719
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/logs
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:17.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:17.072031 14957 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:17.073 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:17.073256 14957 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:17.084 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.083784 14965 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:17.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.253726 14152 debug-util.cc:398] Leaking SignalData structure 0x7b08000b6b40 after lost signal to thread 14027
14:20:17.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.255501 14152 debug-util.cc:398] Leaking SignalData structure 0x7b08000ac7a0 after lost signal to thread 14155
14:20:17.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:17.442699 14941 heartbeater.cc:499] Master 127.13.213.252:33645 was elected leader, sending a full tablet report...
14:20:17.467 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.466791 14942 heartbeater.cc:646] Failed to heartbeat to 127.13.213.253:36059 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.13.213.253:36059: connect: Connection refused (error 111)
14:20:18.781 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:17.085912 14966 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:18.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:18.780565 14957 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 1.695s user 0.486s sys 0.863s
14:20:18.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:18.782157 14957 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 1.697s user 0.487s sys 0.864s
14:20:18.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:18.487432 14964 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 14957
14:20:18.783 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:18.782912 14967 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Resolving timed out after 1696 milliseconds
14:20:18.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:18.783994 14968 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:18.784 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.783901 14957 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:20:18.788 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.788240 14957 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:18.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.791038 14957 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:18.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.792497 14957 hybrid_clock.cc:648] HybridClock initialized: now 1756736418792454 us; error 55 us; skew 500 ppm
14:20:18.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.793540 14957 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:18.798 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:18 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736418, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.253@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:18.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.800423 14957 init.cc:377] Logged in from keytab as kudu/127.13.213.253@KRBTEST.COM (short username kudu)
14:20:18.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.807039 14957 webserver.cc:480] Webserver started at http://127.13.213.253:33719/ using document root <none> and password file <none>
14:20:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.808349 14957 fs_manager.cc:362] Metadata directory not provided
14:20:18.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.808657 14957 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:18.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.820461 14957 fs_manager.cc:714] Time spent opening directory manager: real 0.007s user 0.006s sys 0.000s
14:20:18.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.826731 14986 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:18.828 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.828090 14957 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:20:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.828526 14957 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal
14:20:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "2a4dc60a14a64f39a707a71d81e6a0a6"
14:20:18.831 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:03 on dist-test-slave-k2rc"
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.831614 14957 fs_report.cc:389] FS layout report
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/wal
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data/data
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:18.832 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:18.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.927883 14957 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:18.934 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.933977 14957 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:18.935 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:18.934494 14957 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:19.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.030015 14957 rpc_server.cc:307] RPC server started. Bound to: 127.13.213.253:36059
14:20:19.030 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.030099 15042 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.213.253:36059 every 8 connection(s)
14:20:19.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.034498 14957 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-1/data/info.pb
14:20:19.035 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.035519 14167 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 14957
14:20:19.036 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:20:19.036 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.213.252:33645
14:20:19.037 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.038858 14167 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/logs
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data/info.pb
14:20:19.039 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.252:33645
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.252
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43179
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.252
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:19.040 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:20:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:20:19.041 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:20:19.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.062435 15043 sys_catalog.cc:263] Verifying existing consensus state
14:20:19.071 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.071174 15043 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: Bootstrap starting.
14:20:19.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.128947 15043 log.cc:826] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: Log is configured to *not* fsync() on all Append() calls
14:20:19.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.151299 15043 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:20:19.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.152432 15043 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: Bootstrap complete.
14:20:19.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.192152 15043 raft_consensus.cc:357] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:19.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.195693 15043 raft_consensus.cc:738] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2a4dc60a14a64f39a707a71d81e6a0a6, State: Initialized, Role: FOLLOWER
14:20:19.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.196748 15043 consensus_queue.cc:260] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.208406 15048 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:19.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.209329 15048 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: This master's current role is: FOLLOWER
14:20:19.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.210672 15043 sys_catalog.cc:564] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: configured and running, proceeding with master startup.
14:20:19.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.252697 15060 catalog_manager.cc:1261] Loaded cluster ID: fa90510be2dc4b91a9cfe573748fb8f2
14:20:19.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.253022 15060 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: loading cluster ID for follower catalog manager: success
14:20:19.280 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.279987 15060 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: acquiring CA information for follower catalog manager: success
14:20:19.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.288684 15060 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:20:19.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.260279 14957 master_runner.cc:186] Error getting master registration for 127.13.213.254:36127: Network error: Client connection negotiation failed: client connection to 127.13.213.254:36127: connect: Connection refused (error 111),
14:20:19.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.300495 14957 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:20:19.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.470180 15044 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.470903 15044 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:19.471 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.471644 15044 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:19.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.537856 15044 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:20:19.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.538254 15044 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:20:19.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.538780 15044 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:19.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.539258 15044 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:20:19.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.539630 15044 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:20:19.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.539996 15044 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:20:19.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.540270 15044 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41679/jwks.json
14:20:19.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.553521 14942 heartbeater.cc:344] Connected to a master server at 127.13.213.253:36059
14:20:19.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.561386 15007 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "3a04c7b44f494e3e84a5f643a35bf3c5" instance_seqno: 1756736416247949) as {username='kudu', principal='kudu/127.13.213.195@KRBTEST.COM'} at 127.13.213.195:55895; Asking this server to re-register.
14:20:19.564 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.563339 14942 heartbeater.cc:461] Registering TS with master...
14:20:19.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.567116 15006 ts_manager.cc:194] Registered new tserver with Master: 3a04c7b44f494e3e84a5f643a35bf3c5 (127.13.213.195:33201)
14:20:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.599821 15044 master_runner.cc:386] Master server non-default flags:
14:20:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data
14:20:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal
14:20:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:20:19.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.252
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.252:33645
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:20:19.601 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:20:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:20:19.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:19.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data/info.pb
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.252
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=43179
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/logs
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:20:19.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:20:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:20:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.601547 15044 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:19.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.602918 15044 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:19.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.645659 15072 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:19.908 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.907719 14794 heartbeater.cc:344] Connected to a master server at 127.13.213.253:36059
14:20:19.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.911326 15006 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "097ba40d6e064c87afbe3cb2ad79f915" instance_seqno: 1756736413679283) as {username='kudu', principal='kudu/127.13.213.194@KRBTEST.COM'} at 127.13.213.194:48335; Asking this server to re-register.
14:20:19.913 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.912912 14794 heartbeater.cc:461] Registering TS with master...
14:20:19.917 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.917145 15006 ts_manager.cc:194] Registered new tserver with Master: 097ba40d6e064c87afbe3cb2ad79f915 (127.13.213.194:41985)
14:20:19.967 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.966131 14646 heartbeater.cc:344] Connected to a master server at 127.13.213.253:36059
14:20:19.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.970211 15006 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "1ada331dc71b4be3978332266014431f" instance_seqno: 1756736410695475) as {username='kudu', principal='kudu/127.13.213.193@KRBTEST.COM'} at 127.13.213.193:51287; Asking this server to re-register.
14:20:19.972 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.972213 14646 heartbeater.cc:461] Registering TS with master...
14:20:19.976 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:19.976227 15006 ts_manager.cc:194] Registered new tserver with Master: 1ada331dc71b4be3978332266014431f (127.13.213.193:40897)
14:20:20.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:20 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736420, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.193@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:20.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:20.474431 14528 init.cc:295] Successfully reacquired a new kerberos TGT
14:20:21.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.077905 15086 raft_consensus.cc:491] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:20:21.079 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.078564 15086 raft_consensus.cc:513] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:21.099 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:21.098991 14991 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.13.213.254:36127: connect: Connection refused (error 111)
14:20:21.111 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.110656 15086 leader_election.cc:290] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9e92c364bd674ed7b7de379369369941 (127.13.213.254:36127), 4edc07c774c448db8853b5fdb7bbe6e9 (127.13.213.252:33645)
14:20:21.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:21.126591 14991 leader_election.cc:336] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9e92c364bd674ed7b7de379369369941 (127.13.213.254:36127): Network error: Client connection negotiation failed: client connection to 127.13.213.254:36127: connect: Connection refused (error 111)
14:20:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:21.127434 14990 leader_election.cc:336] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4edc07c774c448db8853b5fdb7bbe6e9 (127.13.213.252:33645): Network error: Client connection negotiation failed: client connection to 127.13.213.252:33645: connect: Connection refused (error 111)
14:20:21.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.128099 14990 leader_election.cc:304] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 2a4dc60a14a64f39a707a71d81e6a0a6; no voters: 4edc07c774c448db8853b5fdb7bbe6e9, 9e92c364bd674ed7b7de379369369941
14:20:21.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.129339 15086 raft_consensus.cc:2747] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
14:20:21.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:21.040362 15071 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 15044
14:20:21.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:21.683642 15044 thread.cc:641] GCE (cloud detector) Time spent creating pthread: real 2.039s user 0.592s sys 1.159s
14:20:21.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:21.683981 15044 thread.cc:608] GCE (cloud detector) Time spent starting thread: real 2.039s user 0.592s sys 1.159s
14:20:21.684 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:19.647270 15073 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:21.693617 15094 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:21.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.693638 15044 server_base.cc:1047] running on GCE node
14:20:21.695 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.695086 15044 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:21.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.698837 15044 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:21.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.700271 15044 hybrid_clock.cc:648] HybridClock initialized: now 1756736421700166 us; error 97 us; skew 500 ppm
14:20:21.701 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.701398 15044 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:21.707 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:21 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736421, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.252@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:21.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.709795 15044 init.cc:377] Logged in from keytab as kudu/127.13.213.252@KRBTEST.COM (short username kudu)
14:20:21.716 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.715982 15044 webserver.cc:480] Webserver started at http://127.13.213.252:43179/ using document root <none> and password file <none>
14:20:21.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.716936 15044 fs_manager.cc:362] Metadata directory not provided
14:20:21.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.717164 15044 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:21.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.725847 15044 fs_manager.cc:714] Time spent opening directory manager: real 0.006s user 0.007s sys 0.000s
14:20:21.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.730430 15101 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:21.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.731528 15044 fs_manager.cc:730] Time spent opening block manager: real 0.004s user 0.005s sys 0.000s
14:20:21.733 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.731854 15044 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "4edc07c774c448db8853b5fdb7bbe6e9"
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:05 on dist-test-slave-k2rc"
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.733948 15044 fs_report.cc:389] FS layout report
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/wal
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data/data
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:21.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:21.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:21.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:21.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.789332 15044 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:21.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.795718 15044 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:21.796 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.796239 15044 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.908818 15044 rpc_server.cc:307] RPC server started. Bound to: 127.13.213.252:33645
14:20:21.909 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.908998 15153 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.213.252:33645 every 8 connection(s)
14:20:21.912 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.912171 15044 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-2/data/info.pb
14:20:21.921 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.921353 14167 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 15044
14:20:21.922 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:20:21.922 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.13.213.254:36127
14:20:21.922 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 0
}
}
14:20:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.924809 14167 external_mini_cluster.cc:1366] Running /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu
14:20:21.925 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/logs
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data/info.pb
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.254:36127
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.254
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38441
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.254
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=test-admin
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=test-user
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=true
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:21.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:21.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:21.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:21.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:21.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:21.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:21.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:20:21.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_require_spnego=false
14:20:21.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10 with env {KRB5CCNAME=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5cc,KRB5_CONFIG=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/krb5.conf,KRB5_KDC_PROFILE=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kdc.conf,KUDU_ENABLE_KRB5_REALM_FIX=yes}
14:20:21.927 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.925694 15154 sys_catalog.cc:263] Verifying existing consensus state
14:20:21.933 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.933276 15154 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: Bootstrap starting.
14:20:21.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:21 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736420, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.193@KRBTEST.COM for kudu/127.13.213.252@KRBTEST.COM
14:20:21.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:21.997323 15154 log.cc:826] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: Log is configured to *not* fsync() on all Append() calls
14:20:22.006 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.005684 14645 heartbeater.cc:344] Connected to a master server at 127.13.213.252:33645
14:20:22.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.018779 15154 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:20:22.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.019686 15154 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: Bootstrap complete.
14:20:22.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.046401 15154 raft_consensus.cc:357] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:22.049 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.049567 15154 raft_consensus.cc:738] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4edc07c774c448db8853b5fdb7bbe6e9, State: Initialized, Role: FOLLOWER
14:20:22.051 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.050580 15154 consensus_queue.cc:260] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:22.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.065222 15154 sys_catalog.cc:564] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: configured and running, proceeding with master startup.
14:20:22.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.064679 15161 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:22.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.066663 15161 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: This master's current role is: FOLLOWER
14:20:22.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.193912 15173 catalog_manager.cc:1261] Loaded cluster ID: fa90510be2dc4b91a9cfe573748fb8f2
14:20:22.196 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.194206 15173 catalog_manager.cc:1554] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: loading cluster ID for follower catalog manager: success
14:20:22.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.217173 15173 catalog_manager.cc:1576] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: acquiring CA information for follower catalog manager: success
14:20:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.175653 15044 master_runner.cc:186] Error getting master registration for 127.13.213.254:36127: Network error: Client connection negotiation failed: client connection to 127.13.213.254:36127: connect: Connection refused (error 111),
14:20:22.231 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.231280 15044 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:20:22.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.232789 15173 catalog_manager.cc:1604] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
14:20:22.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.434039 15156 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
14:20:22.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.434834 15156 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
14:20:22.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.435925 15156 flags.cc:425] Enabled unsafe flag: --never_fsync=true
14:20:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.493813 15156 flags.cc:425] Enabled experimental flag: --authn_token_validity_seconds=10
14:20:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.494141 15156 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
14:20:22.494 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.494621 15156 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
14:20:22.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.494959 15156 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
14:20:22.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.495293 15156 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
14:20:22.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.495610 15156 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
14:20:22.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.495851 15156 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41679/jwks.json
14:20:22.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.540200 15156 master_runner.cc:386] Master server non-default flags:
14:20:22.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
14:20:22.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.13.213.212:37643
14:20:22.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
14:20:22.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
14:20:22.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data
14:20:22.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal
14:20:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --authn_token_validity_seconds=10
14:20:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
14:20:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.13.213.254:36127,127.13.213.253:36059,127.13.213.252:33645
14:20:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --keytab_file=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/krb5kdc/kudu.keytab
14:20:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --principal=kudu/127.13.213.254
14:20:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
14:20:22.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.13.213.254:36127
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41679/jwks.json
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_authentication=required
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --superuser_acl=<redacted>
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --user_acl=<redacted>
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data/info.pb
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.13.213.254
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=38441
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/logs
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.19.0-SNAPSHOT
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision 06466b25e294b4d7d455e6c56263318dc34a80ef
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by None at 01 Sep 2025 14:08:12 UTC on 24a791456cd2
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 7849
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) TSAN enabled
14:20:22.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.541837 15156 env_posix.cc:2264] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
14:20:22.543 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.543192 15156 file_cache.cc:492] Constructed file cache file cache with capacity 419430
14:20:22.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.554178 15182 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.606125 14941 heartbeater.cc:344] Connected to a master server at 127.13.213.252:33645
14:20:22.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.606642 14941 heartbeater.cc:499] Master 127.13.213.252:33645 was elected leader, sending a full tablet report...
14:20:22.614 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.613999 15118 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "3a04c7b44f494e3e84a5f643a35bf3c5" instance_seqno: 1756736416247949) as {username='kudu', principal='kudu/127.13.213.195@KRBTEST.COM'} at 127.13.213.195:38387; Asking this server to re-register.
14:20:22.616 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.615911 14941 heartbeater.cc:461] Registering TS with master...
14:20:22.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.619783 15118 ts_manager.cc:194] Registered new tserver with Master: 3a04c7b44f494e3e84a5f643a35bf3c5 (127.13.213.195:33201)
14:20:22.963 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.963439 15194 raft_consensus.cc:491] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
14:20:22.984 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:22.963886 15194 raft_consensus.cc:513] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:23.008 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:23.007606 14991 leader_election.cc:336] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 9e92c364bd674ed7b7de379369369941 (127.13.213.254:36127): Network error: Client connection negotiation failed: client connection to 127.13.213.254:36127: connect: Connection refused (error 111)
14:20:23.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.011061 15194 leader_election.cc:290] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 9e92c364bd674ed7b7de379369369941 (127.13.213.254:36127), 4edc07c774c448db8853b5fdb7bbe6e9 (127.13.213.252:33645)
14:20:23.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.012233 14793 heartbeater.cc:344] Connected to a master server at 127.13.213.252:33645
14:20:23.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.012329 15118 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "1ada331dc71b4be3978332266014431f" instance_seqno: 1756736410695475) as {username='kudu', principal='kudu/127.13.213.193@KRBTEST.COM'} at 127.13.213.193:51589; Asking this server to re-register.
14:20:23.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.014410 14645 heartbeater.cc:461] Registering TS with master...
14:20:23.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.014992 15118 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "097ba40d6e064c87afbe3cb2ad79f915" instance_seqno: 1756736413679283) as {username='kudu', principal='kudu/127.13.213.194@KRBTEST.COM'} at 127.13.213.194:43263; Asking this server to re-register.
14:20:23.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.016166 14793 heartbeater.cc:461] Registering TS with master...
14:20:23.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.019227 15118 ts_manager.cc:194] Registered new tserver with Master: 1ada331dc71b4be3978332266014431f (127.13.213.193:40897)
14:20:23.022 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.021801 15118 ts_manager.cc:194] Registered new tserver with Master: 097ba40d6e064c87afbe3cb2ad79f915 (127.13.213.194:41985)
14:20:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.032235 15128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" is_pre_election: true
14:20:23.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.033278 15128 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 2a4dc60a14a64f39a707a71d81e6a0a6 in term 1.
14:20:23.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.035086 14990 leader_election.cc:304] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2a4dc60a14a64f39a707a71d81e6a0a6, 4edc07c774c448db8853b5fdb7bbe6e9; no voters: 9e92c364bd674ed7b7de379369369941
14:20:23.036 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.036422 15194 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Leader pre-election won for term 2
14:20:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.037014 15194 raft_consensus.cc:491] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
14:20:23.037 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.037532 15194 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 1 FOLLOWER]: Advancing to term 2
14:20:23.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.057582 15194 raft_consensus.cc:513] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:23.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.067058 15128 tablet_service.cc:1813] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" candidate_term: 2 candidate_status { last_received { term: 1 index: 4 } } ignore_live_leader: false dest_uuid: "4edc07c774c448db8853b5fdb7bbe6e9"
14:20:23.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.067703 15128 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 1 FOLLOWER]: Advancing to term 2
14:20:23.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:23.070307 14991 leader_election.cc:336] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 9e92c364bd674ed7b7de379369369941 (127.13.213.254:36127): Network error: Client connection negotiation failed: client connection to 127.13.213.254:36127: connect: Connection refused (error 111)
14:20:23.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.077461 15194 leader_election.cc:290] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [CANDIDATE]: Term 2 election: Requested vote from peers 9e92c364bd674ed7b7de379369369941 (127.13.213.254:36127), 4edc07c774c448db8853b5fdb7bbe6e9 (127.13.213.252:33645)
14:20:23.081 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.080579 15128 raft_consensus.cc:2466] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 2a4dc60a14a64f39a707a71d81e6a0a6 in term 2.
14:20:23.087 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.086414 14990 leader_election.cc:304] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 2a4dc60a14a64f39a707a71d81e6a0a6, 4edc07c774c448db8853b5fdb7bbe6e9; no voters: 9e92c364bd674ed7b7de379369369941
14:20:23.088 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.087844 15194 raft_consensus.cc:2802] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 2 FOLLOWER]: Leader election won for term 2
14:20:23.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.095589 15194 raft_consensus.cc:695] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [term 2 LEADER]: Becoming Leader. State: Replica: 2a4dc60a14a64f39a707a71d81e6a0a6, State: Running, Role: LEADER
14:20:23.097 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.096755 15194 consensus_queue.cc:237] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.135550 15200 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2a4dc60a14a64f39a707a71d81e6a0a6. Latest consensus state: current_term: 2 leader_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:23.136 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.136713 15200 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: This master's current role is: LEADER
14:20:23.151 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.151439 15202 catalog_manager.cc:1477] Loading table and tablet metadata into memory...
14:20:23.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.210790 15202 catalog_manager.cc:1486] Initializing Kudu cluster ID...
14:20:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.231519 15202 catalog_manager.cc:1261] Loaded cluster ID: fa90510be2dc4b91a9cfe573748fb8f2
14:20:23.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.234002 15202 catalog_manager.cc:1497] Initializing Kudu internal certificate authority...
14:20:23.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.250164 15202 catalog_manager.cc:1506] Loading token signing keys...
14:20:23.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.271931 15202 catalog_manager.cc:5966] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6: Loaded TSK: 0
14:20:23.276 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.274626 15202 catalog_manager.cc:1516] Initializing in-progress tserver states...
14:20:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:23 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736423, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.194@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:23.480 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.479804 14676 init.cc:295] Successfully reacquired a new kerberos TGT
14:20:23.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:23.533224 14991 consensus_peers.cc:489] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 -> Peer 9e92c364bd674ed7b7de379369369941 (127.13.213.254:36127): Couldn't send request to peer 9e92c364bd674ed7b7de379369369941. Status: Network error: Client connection negotiation failed: client connection to 127.13.213.254:36127: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
14:20:23.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.709100 15128 raft_consensus.cc:1273] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [term 2 FOLLOWER]: Refusing update from remote peer 2a4dc60a14a64f39a707a71d81e6a0a6: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 2 index: 5. (index mismatch)
14:20:23.715 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.712275 15200 consensus_queue.cc:1035] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
14:20:23.790 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.789167 15204 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2a4dc60a14a64f39a707a71d81e6a0a6. Latest consensus state: current_term: 2 leader_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:23.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.790799 15204 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: This master's current role is: FOLLOWER
14:20:23.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:23.797009 13590 debug-util.cc:398] Leaking SignalData structure 0x7b0800093cc0 after lost signal to thread 13525
14:20:23.799 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:23.797746 13590 debug-util.cc:398] Leaking SignalData structure 0x7b08000a35e0 after lost signal to thread 13593
14:20:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.812577 15200 sys_catalog.cc:455] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:23.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.813486 15200 sys_catalog.cc:458] T 00000000000000000000000000000000 P 2a4dc60a14a64f39a707a71d81e6a0a6 [sys.catalog]: This master's current role is: LEADER
14:20:23.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.816649 15204 sys_catalog.cc:455] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:23.819 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:23.818969 15204 sys_catalog.cc:458] T 00000000000000000000000000000000 P 4edc07c774c448db8853b5fdb7bbe6e9 [sys.catalog]: This master's current role is: FOLLOWER
14:20:24.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:24.304214 14957 master_runner.cc:186] Error getting master registration for 127.13.213.254:36127: Network error: Client connection negotiation failed: client connection to 127.13.213.254:36127: connect: Connection refused (error 111),
14:20:24.308 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:24.304658 14957 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
14:20:24.590 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:24.589176 14942 heartbeater.cc:499] Master 127.13.213.253:36059 was elected leader, sending a full tablet report...
14:20:24.942 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:24.941540 14794 heartbeater.cc:499] Master 127.13.213.253:36059 was elected leader, sending a full tablet report...
14:20:25.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.009712 14646 heartbeater.cc:499] Master 127.13.213.253:36059 was elected leader, sending a full tablet report...
14:20:25.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:23.956183 15181 debug-util.cc:398] Leaking SignalData structure 0x7b0800037200 after lost signal to thread 15156
14:20:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:25.210975 15156 thread.cc:641] OpenStack (cloud detector) Time spent creating pthread: real 2.655s user 0.699s sys 1.142s
14:20:25.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:25.213762 15156 thread.cc:608] OpenStack (cloud detector) Time spent starting thread: real 2.658s user 0.700s sys 1.142s
14:20:25.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:25.214578 15185 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:25.220 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.218454 15156 server_base.cc:1042] Not found: could not retrieve instance metadata: unable to detect cloud type of this node, probably running in non-cloud environment
14:20:25.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:25.212618 15184 instance_detector.cc:116] could not retrieve GCE instance metadata: Timed out: curl timeout: Timeout was reached: Connection timed out after 2655 milliseconds
14:20:25.226 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:22.555732 15183 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
14:20:25.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.228720 15156 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
14:20:25.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.248899 15156 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
14:20:25.250 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.250382 15156 hybrid_clock.cc:648] HybridClock initialized: now 1756736425250339 us; error 42 us; skew 500 ppm
14:20:25.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.251607 15156 server_base.cc:847] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
14:20:25.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:25 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736425, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.254@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:25.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.259929 15156 init.cc:377] Logged in from keytab as kudu/127.13.213.254@KRBTEST.COM (short username kudu)
14:20:25.302 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.302515 15156 webserver.cc:480] Webserver started at http://127.13.213.254:38441/ using document root <none> and password file <none>
14:20:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.306267 15156 fs_manager.cc:362] Metadata directory not provided
14:20:25.310 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.306596 15156 fs_manager.cc:368] Using write-ahead log directory (fs_wal_dir) as metadata directory
14:20:25.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.331034 15156 fs_manager.cc:714] Time spent opening directory manager: real 0.020s user 0.012s sys 0.008s
14:20:25.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.337374 15226 log_block_manager.cc:3788] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
14:20:25.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.338747 15156 fs_manager.cc:730] Time spent opening block manager: real 0.005s user 0.004s sys 0.000s
14:20:25.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.339359 15156 fs_manager.cc:647] Opened local filesystem: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data,/tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal
14:20:25.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "9e92c364bd674ed7b7de379369369941"
14:20:25.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2025-09-01 14:20:01 on dist-test-slave-k2rc"
14:20:25.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.342197 15156 fs_report.cc:389] FS layout report
14:20:25.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
14:20:25.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal
14:20:25.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/wal
14:20:25.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data/data
14:20:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
14:20:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
14:20:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
14:20:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
14:20:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
14:20:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
14:20:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
14:20:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
14:20:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
14:20:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
14:20:25.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
14:20:25.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.423847 15156 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
14:20:25.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.432150 15156 env_posix.cc:2264] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
14:20:25.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.432730 15156 kserver.cc:163] Server-wide thread pool size limit: 3276
14:20:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.591080 15280 acceptor_pool.cc:272] collecting diagnostics on the listening RPC socket 127.13.213.254:36127 every 8 connection(s)
14:20:25.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.591619 15156 rpc_server.cc:307] RPC server started. Bound to: 127.13.213.254:36127
14:20:25.595 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.595345 15156 server_base.cc:1179] Dumped server information to /tmp/dist-test-taskciA_xo/test-tmp/mini-kudu-cluster1735978025677896847/master-0/data/info.pb
14:20:25.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.602859 14167 external_mini_cluster.cc:1428] Started /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu as pid 15156
14:20:25.603 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
14:20:25.618 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.618502 15281 sys_catalog.cc:263] Verifying existing consensus state
14:20:25.626 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.626175 15281 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: Bootstrap starting.
14:20:25.699 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.699399 15281 log.cc:826] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: Log is configured to *not* fsync() on all Append() calls
14:20:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:25.717772 15285 negotiation.cc:337] Failed RPC negotiation. Trace:
14:20:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:25.666939 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.13.213.195:40151 (local address 127.13.213.254:36127)
14:20:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:25.688043 (+ 21104us) server_negotiation.cc:206] Beginning negotiation
14:20:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:25.688084 (+ 41us) server_negotiation.cc:399] Waiting for connection header
14:20:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:25.691556 (+ 3472us) server_negotiation.cc:407] Connection header received
14:20:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:25.702099 (+ 10543us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:20:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:25.702163 (+ 64us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:20:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:25.702730 (+ 567us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:20:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:25.703235 (+ 505us) server_negotiation.cc:226] Negotiated authn=SASL
14:20:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:25.706300 (+ 3065us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.13.213.195:40151: BlockingRecv error: recv got EOF from 127.13.213.195:40151 (error 108)
14:20:25.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":20075,"thread_start_us":20828,"threads_started":1}
14:20:25.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.741993 15281 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=3 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
14:20:25.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.743005 15281 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941: Bootstrap complete.
14:20:25.758 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.213.253:36059(127.13.213.253:36059)] connecting to peer
14:20:25.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.773134 15281 raft_consensus.cc:357] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:25.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.776291 15281 raft_consensus.cc:738] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9e92c364bd674ed7b7de379369369941, State: Initialized, Role: FOLLOWER
14:20:25.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.777344 15281 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } }
14:20:25.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.785892 15287 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:25.786 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.786674 15287 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: This master's current role is: FOLLOWER
14:20:25.789 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.789438 15281 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: configured and running, proceeding with master startup.
14:20:25.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.872426 15298 catalog_manager.cc:797] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
14:20:25.895 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:25 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736425, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.254@KRBTEST.COM for kudu/127.13.213.254@KRBTEST.COM
14:20:25.932 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:25 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736425, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.254@KRBTEST.COM for kudu/127.13.213.253@KRBTEST.COM
14:20:25.954 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.213.252:33645(127.13.213.252:33645)] connecting to peer
14:20:25.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:25 dist-test-slave-k2rc krb5kdc[14180](info): AS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736425, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.195@KRBTEST.COM for krbtgt/KRBTEST.COM@KRBTEST.COM
14:20:25.960 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:25.960359 14823 init.cc:295] Successfully reacquired a new kerberos TGT
14:20:25.968 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.13.213.254:36127(127.13.213.254:36127)] connecting to peer
14:20:25.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:25 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736425, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.254@KRBTEST.COM for kudu/127.13.213.252@KRBTEST.COM
14:20:26.008 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.13.213.252:33645(127.13.213.252:33645)] Successfully connected to peer
14:20:26.012 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.13.213.253:36059(127.13.213.253:36059)] Successfully connected to peer
14:20:26.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:26 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736423, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.194@KRBTEST.COM for kudu/127.13.213.254@KRBTEST.COM
14:20:26.055 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.054476 14795 heartbeater.cc:344] Connected to a master server at 127.13.213.254:36127
14:20:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.061504 15243 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "097ba40d6e064c87afbe3cb2ad79f915" instance_seqno: 1756736413679283) as {username='kudu', principal='kudu/127.13.213.194@KRBTEST.COM'} at 127.13.213.194:52733; Asking this server to re-register.
14:20:26.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.063254 14795 heartbeater.cc:461] Registering TS with master...
14:20:26.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.066826 15243 ts_manager.cc:194] Registered new tserver with Master: 097ba40d6e064c87afbe3cb2ad79f915 (127.13.213.194:41985)
14:20:26.095 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:26 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736420, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.193@KRBTEST.COM for kudu/127.13.213.254@KRBTEST.COM
14:20:26.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.117547 14650 heartbeater.cc:344] Connected to a master server at 127.13.213.254:36127
14:20:26.123 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.121717 15243 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "1ada331dc71b4be3978332266014431f" instance_seqno: 1756736410695475) as {username='kudu', principal='kudu/127.13.213.193@KRBTEST.COM'} at 127.13.213.193:38265; Asking this server to re-register.
14:20:26.127 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.125636 14650 heartbeater.cc:461] Registering TS with master...
14:20:26.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.129524 15243 ts_manager.cc:194] Registered new tserver with Master: 1ada331dc71b4be3978332266014431f (127.13.213.193:40897)
14:20:26.145 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.13.213.254:36127(127.13.213.254:36127)] Successfully connected to peer
14:20:26.178 [WARN - kudu-nio-1] (SecurityContext.java:206) Caller-provided Subject has a Kerberos ticket that is about to expire. Kudu expects the application to renew or re-acquire its own tickets before expiration.
14:20:26.182 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.213.253:36059(127.13.213.253:36059)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:20:26.183 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.213.253:36059: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:20:26.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:26.186275 15300 negotiation.cc:337] Failed RPC negotiation. Trace:
14:20:26.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.011145 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:51830 (local address 127.13.213.253:36059)
14:20:26.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.011903 (+ 758us) server_negotiation.cc:206] Beginning negotiation
14:20:26.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.011929 (+ 26us) server_negotiation.cc:399] Waiting for connection header
14:20:26.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.110245 (+ 98316us) server_negotiation.cc:407] Connection header received
14:20:26.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.143965 (+ 33720us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:20:26.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.144015 (+ 50us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:20:26.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.144273 (+ 258us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:20:26.186 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.144433 (+ 160us) server_negotiation.cc:226] Negotiated authn=SASL
14:20:26.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.184877 (+ 40444us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:51830: BlockingRecv error: recv got EOF from 127.0.0.1:51830 (error 108)
14:20:26.187 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":145}
14:20:26.188 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.13.213.254:36127(127.13.213.254:36127)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:20:26.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.13.213.252:33645(127.13.213.252:33645)] cleaning up while in state NEGOTIATING due to: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:20:26.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.13.213.252:33645: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:20:26.190 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.13.213.252:33645(127.13.213.252:33645)] handling channelInactive
14:20:26.191 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.13.213.254:36127: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available
14:20:26.192 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:26.192241 15305 negotiation.cc:337] Failed RPC negotiation. Trace:
14:20:26.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.000813 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:37024 (local address 127.13.213.252:33645)
14:20:26.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.006873 (+ 6060us) server_negotiation.cc:206] Beginning negotiation
14:20:26.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.006906 (+ 33us) server_negotiation.cc:399] Waiting for connection header
14:20:26.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.110053 (+103147us) server_negotiation.cc:407] Connection header received
14:20:26.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.143982 (+ 33929us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:20:26.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.144014 (+ 32us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:20:26.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.144268 (+ 254us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:20:26.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.144425 (+ 157us) server_negotiation.cc:226] Negotiated authn=SASL
14:20:26.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.190490 (+ 46065us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:37024: BlockingRecv error: recv got EOF from 127.0.0.1:37024 (error 108)
14:20:26.195 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":5348,"thread_start_us":5080,"threads_started":1}
14:20:26.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Sep 01 14:20:26 dist-test-slave-k2rc krb5kdc[14180](info): TGS_REQ (2 etypes {17 16}) 127.0.0.1: ISSUE: authtime 1756736418, etypes {rep=17 tkt=17 ses=17}, kudu/127.13.213.253@KRBTEST.COM for kudu/127.13.213.254@KRBTEST.COM
14:20:26.197 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1523) Notify RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=589), Traces: [0ms] refreshing cache from master, [90ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.213.253:36059, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.213.252:33645, [316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.213.254:36127, [531ms] Sub RPC ConnectToMaster: received response from server master-127.13.213.253:36059: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [536ms] Sub RPC ConnectToMaster: received response from server master-127.13.213.254:36127: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [537ms] Sub RPC ConnectToMaster: received response from server master-127.13.213.252:33645: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) after lookup exception
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.13.213.253:36059,127.13.213.252:33645,127.13.213.254:36127). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) [main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) [main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) [main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) [async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) [async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) [main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) [main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) [main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) [main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) [main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) [main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) [main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) [netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) [netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) [netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.13.213.253:36059,127.13.213.252:33645,127.13.213.254:36127). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14:20:26.210 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ListTabletServers, tablet=null, attempt=1, TimeoutTracker(timeout=30000, elapsed=601), Traces: [0ms] refreshing cache from master, [90ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.213.253:36059, [302ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.213.252:33645, [316ms] Sub RPC ConnectToMaster: sending RPC to server master-127.13.213.254:36127, [531ms] Sub RPC ConnectToMaster: received response from server master-127.13.213.253:36059: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [536ms] Sub RPC ConnectToMaster: received response from server master-127.13.213.254:36127: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, [537ms] Sub RPC ConnectToMaster: received response from server master-127.13.213.252:33645: Not authorized: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, deferred=null) with no deferred attached!
14:20:26.213 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.212976 15253 raft_consensus.cc:3058] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [term 1 FOLLOWER]: Advancing to term 2
14:20:26.221 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.213.254:36127(127.13.213.254:36127)] disconnecting while in state TERMINATED
14:20:26.222 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.213.253:36059(127.13.213.253:36059)] disconnecting while in state TERMINATED
14:20:26.222 [DEBUG - Time-limited test] (Connection.java:575) [peer master-127.13.213.252:33645(127.13.213.252:33645)] disconnecting while in state TERMINATED
14:20:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:26.228947 15306 negotiation.cc:337] Failed RPC negotiation. Trace:
14:20:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.011843 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:43474 (local address 127.13.213.254:36127)
14:20:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.026724 (+ 14881us) server_negotiation.cc:206] Beginning negotiation
14:20:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.026757 (+ 33us) server_negotiation.cc:399] Waiting for connection header
14:20:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.145685 (+118928us) server_negotiation.cc:407] Connection header received
14:20:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.146544 (+ 859us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
14:20:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.146581 (+ 37us) server_negotiation.cc:461] Received NEGOTIATE request from client
14:20:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.146826 (+ 245us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
14:20:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.146995 (+ 169us) server_negotiation.cc:226] Negotiated authn=SASL
14:20:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0901 14:20:26.227010 (+ 80015us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:43474: BlockingRecv error: recv got EOF from 127.0.0.1:43474 (error 108)
14:20:26.229 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":14242,"thread_start_us":13949,"threads_started":1}
14:20:26.230 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.13.213.253:36059(127.13.213.253:36059)] handling channelInactive
14:20:26.233 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:20:26.242 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:20:26.244 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.13.213.254:36127(127.13.213.254:36127)] handling channelInactive
14:20:26.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.252781 15313 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:26.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.253865 15313 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: This master's current role is: FOLLOWER
14:20:26.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.257524 15287 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 2a4dc60a14a64f39a707a71d81e6a0a6. Latest consensus state: current_term: 2 leader_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "9e92c364bd674ed7b7de379369369941" member_type: VOTER last_known_addr { host: "127.13.213.254" port: 36127 } } peers { permanent_uuid: "2a4dc60a14a64f39a707a71d81e6a0a6" member_type: VOTER last_known_addr { host: "127.13.213.253" port: 36059 } } peers { permanent_uuid: "4edc07c774c448db8853b5fdb7bbe6e9" member_type: VOTER last_known_addr { host: "127.13.213.252" port: 33645 } } }
14:20:26.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.258479 15287 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9e92c364bd674ed7b7de379369369941 [sys.catalog]: This master's current role is: FOLLOWER
14:20:26.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.264317 14167 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 14512
14:20:26.298 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.298187 14167 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 14659
14:20:26.331 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.331234 14167 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 14807
14:20:26.366 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.366241 14167 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 15156
14:20:26.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.394140 14167 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 14957
14:20:26.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:26.436462 14167 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 15044
14:20:26.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:26.632359 13868 debug-util.cc:398] Leaking SignalData structure 0x7b08000b9c20 after lost signal to thread 13741
14:20:26.634 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20250901 14:20:26.634044 13868 debug-util.cc:398] Leaking SignalData structure 0x7b080003cc60 after lost signal to thread 13871
14:20:27.820 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-01T14:20:27Z chronyd exiting
14:20:28.866 [DEBUG - main] (AsyncKuduClient.java:2704) Releasing all remaining resources
14:20:28.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:28.867487 13439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 13720
14:20:28.898 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:28.898169 13439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 13876
14:20:28.926 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:28.925697 13439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 14026
14:20:28.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:28.952072 13439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 13451
14:20:28.979 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:28.979250 13439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 13521
14:20:29.013 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20250901 14:20:29.012984 13439 external_mini_cluster.cc:1658] Killing /tmp/dist-test-taskciA_xo/build/tsan/bin/kudu with pid 13596
14:20:30.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2025-09-01T14:20:30Z chronyd exiting
14:20:31.211 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.13.213.253:36059,127.13.213.252:33645,127.13.213.254:36127). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.13.213.253:36059,127.13.213.252:33645,127.13.213.254:36127). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
14:20:31.917 [ERROR - main] (RetryRule.java:253) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: giving up after 1 attempts
14:20:31.917 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally: failed attempt 1
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.13.213.253:36059,127.13.213.252:33645,127.13.213.254:36127). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110) ~[main/:?]
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564) ~[main/:?]
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263) ~[main/:?]
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158) ~[test/:?]
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_275]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_275]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_275]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_275]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376) ~[main/:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259) ~[async-1.4.1.jar:?]
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002) ~[async-1.4.1.jar:?]
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277) ~[main/:?]
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331) ~[main/:?]
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256) ~[main/:?]
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158) ~[main/:?]
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154) ~[main/:?]
at org.apache.kudu.client.Connection.cleanup(Connection.java:737) ~[main/:?]
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455) ~[main/:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[netty-handler-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[netty-codec-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[netty-transport-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[netty-common-4.1.115.Final.jar:4.1.115.Final]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
org.apache.kudu.client.NonRecoverableException: Couldn't find a valid master in (127.13.213.253:36059,127.13.213.252:33645,127.13.213.254:36127). Exceptions received: [org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available, org.apache.kudu.client.NonRecoverableException: server requires authentication, but client Kerberos credentials (TGT) have expired. Authentication tokens were not used because no token is available]
at org.apache.kudu.client.KuduException.transformException(KuduException.java:110)
at org.apache.kudu.client.KuduClient.joinAndHandleException(KuduClient.java:564)
at org.apache.kudu.client.KuduClient.listTabletServers(KuduClient.java:263)
at org.apache.kudu.client.TestSecurity.checkClientCanReconnect(TestSecurity.java:158)
at org.apache.kudu.client.TestSecurity.testExternallyProvidedSubjectRefreshedExternally(TestSecurity.java:454)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:748)
Suppressed: org.apache.kudu.client.KuduException$OriginalException: Original asynchronous stack trace
at org.apache.kudu.client.ConnectToCluster.incrementCountAndCheckExhausted(ConnectToCluster.java:268)
at org.apache.kudu.client.ConnectToCluster.access$000(ConnectToCluster.java:51)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:387)
at org.apache.kudu.client.ConnectToCluster$ConnectToMasterErrCB.call(ConnectToCluster.java:376)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1280)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.handleContinuation(Deferred.java:1315)
at com.stumbleupon.async.Deferred.doCall(Deferred.java:1286)
at com.stumbleupon.async.Deferred.runCallbacks(Deferred.java:1259)
at com.stumbleupon.async.Deferred.callback(Deferred.java:1002)
at org.apache.kudu.client.KuduRpc.handleCallback(KuduRpc.java:277)
at org.apache.kudu.client.KuduRpc.errback(KuduRpc.java:331)
at org.apache.kudu.client.RpcProxy.responseReceived(RpcProxy.java:256)
at org.apache.kudu.client.RpcProxy.access$000(RpcProxy.java:64)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:158)
at org.apache.kudu.client.RpcProxy$1.call(RpcProxy.java:154)
at org.apache.kudu.client.Connection.cleanup(Connection.java:737)
at org.apache.kudu.client.Connection.exceptionCaught(Connection.java:455)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:325)
at io.netty.channel.AbstractChannelHandlerContext.fireExceptionCaught(AbstractChannelHandlerContext.java:317)
at io.netty.channel.ChannelInboundHandlerAdapter.exceptionCaught(ChannelInboundHandlerAdapter.java:143)
at io.netty.channel.AbstractChannelHandlerContext.invokeExceptionCaught(AbstractChannelHandlerContext.java:346)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:447)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more